diff --git a/CHANGES.txt b/CHANGES.txt index f0895d9f3..ee400ac2a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,15 @@ Current (7.13.0) New: Added OpenRewrite to the build with a hand-picked recipe list (see rewrite.yml), and applied it to the main sources (Julien Herr) Fixed: Remove leftover dead JUnit code: the deprecated unused ConversionUtils and orphaned JUnit test samples, following the removal of JUnit execution support in 7.10.0 (Julien Herr) +New: Extract the command line front end out of testng-core into the new testng-cli and testng-jcommander modules, so that testng-core no longer depends on JCommander (Julien Herr) + - org.testng.TestNG.main now delegates to an org.testng.ITestNGCliRunner looked up through the ServiceLoader. The reference implementation is bundled inside the org.testng:testng jar, so "java -cp testng.jar org.testng.TestNG suite.xml" is unchanged. Repackaging testng.jar without META-INF/services/org.testng.ITestNGCliRunner now disables the command line. + - testng-core no longer brings org.jcommander:jcommander transitively. Embedders that drive TestNG through its Java API drop a dependency; those that relied on it being on the classpath have to declare it. + - Deprecated: TestNG.main, TestNG.privateMain, org.testng.CommandLineArgs (use org.testng.cli.CliOptions), TestNG.configure(CommandLineArgs) (use org.testng.cli.CliConfigurer) and TestNG.validateCommandLineParameters. They all remain available and are scheduled for removal in 8.0; see the behaviour changes listed below. + - TestNG.validateCommandLineParameters now reports failures as org.testng.TestNGException instead of JCommander's ParameterException. Both are unchecked, so the signature is unchanged, but a caller catching ParameterException no longer catches it. + - TestNG gained setListenerComparatorClass, setListenerFactoryClass, setExecutorServiceFactoryClass and setInjectorFactoryClass, which instantiate through the object factory in use. They are named rather than overloaded so that existing calls passing a bare null keep compiling. + - TestNG.setThreadCount now raises a TestNGException for a value below 1 instead of printing the usage banner and calling System.exit(1). The command line output is unchanged, since TestNG.main turns that exception into the same message and exit code; embedders get an exception they can handle. + - ITestNGCliRunner implementations never terminate the JVM: an unusable command line comes back as a TestNGException. Only TestNG.main turns that into a message, a usage banner and exit code 1, so the observable command line behaviour is unchanged. TestNG.privateMain now throws where it used to exit, which is what an embedder wants. + - Moved: org.testng.Converter is now org.testng.cli.jcommander.Converter. It is still bundled in testng.jar. Update: Dependency refresh: Guice 6.0.0, JCommander 2.0, snakeyaml 2.6, slf4j-api 2.0.18. Guice 7 and JCommander 3 were skipped: they require jakarta.inject and Java 17 respectively Fixed: GITHUB-3242: use-global-thread-pool no longer refuses to start a suite when the number of data-driven tests reaches thread-count (Krishnan Mahadevan) New: GITHUB-3290: Support data providers that return Stream or Stream; the stream is consumed lazily and closed once its rows have been consumed (Krishnan Mahadevan) diff --git a/settings.gradle.kts b/settings.gradle.kts index bf8a29512..f21b05cf0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,9 +22,11 @@ develocity { include(":testng") include(":testng-api") include(":testng-bom") +include(":testng-cli") include(":testng-collections") include(":testng-core") include(":testng-core-api") +include(":testng-jcommander") include(":testng-reflection-utils") include(":testng-runner-api") include(":testng-test-kit") diff --git a/testng-cli/src/main/java/org/testng/cli/AbstractCliRunner.java b/testng-cli/src/main/java/org/testng/cli/AbstractCliRunner.java new file mode 100644 index 000000000..5b01020d9 --- /dev/null +++ b/testng-cli/src/main/java/org/testng/cli/AbstractCliRunner.java @@ -0,0 +1,50 @@ +package org.testng.cli; + +import org.testng.ITestListener; +import org.testng.ITestNGCliRunner; +import org.testng.TestNG; +import org.testng.TestNGException; + +/** + * Base class for command line front ends. Subclasses only have to turn {@code argv} into {@link + * CliOptions} and print a usage banner; validation, configuration and the run itself are shared. + * + *

Nothing here terminates the JVM: a command line that cannot be honoured comes back as a {@link + * CliParseException}, which lets the same code drive a real command line, a test, or an embedding + * process. + * + * @since 7.13 + */ +public abstract class AbstractCliRunner implements ITestNGCliRunner { + + /** + * Parses the raw command line. This is the only thing a front end has to supply. + * + * @param argv the TestNG command line parameters. + * @return the parsed options. + * @throws CliParseException when {@code argv} is not a valid command line. + */ + protected abstract CliOptions parse(String[] argv); + + @Override + public TestNG run(String[] argv, ITestListener listener) { + TestNG result = new TestNG(); + + if (null != listener) { + result.addListener(listener); + } + + CliOptions cli = parse(argv); + CliConfigurer.validate(cli); + CliConfigurer.configure(result, cli); + + try { + result.run(); + } catch (TestNGException ex) { + // A command line reports a broken run as a failing status, not as a propagating exception. + result.reportRunFailure(ex); + } + + return result; + } +} diff --git a/testng-cli/src/main/java/org/testng/cli/CliConfigurer.java b/testng-cli/src/main/java/org/testng/cli/CliConfigurer.java new file mode 100644 index 000000000..fc1b536df --- /dev/null +++ b/testng-cli/src/main/java/org/testng/cli/CliConfigurer.java @@ -0,0 +1,243 @@ +package org.testng.cli; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.testng.IExecutorServiceFactory; +import org.testng.IInjectorFactory; +import org.testng.ITestNGListener; +import org.testng.ITestNGListenerFactory; +import org.testng.ITestObjectFactory; +import org.testng.ITestRunnerFactory; +import org.testng.ListenerComparator; +import org.testng.TestNG; +import org.testng.collections.Lists; +import org.testng.internal.ClassHelper; +import org.testng.internal.Utils; +import org.testng.log4testng.Logger; +import org.testng.xml.XmlSuite; + +/** + * Turns the values a parser collected into {@link CliOptions} into a configured {@link TestNG} + * instance. This is where every "string to object" conversion a command line needs lives: class + * loading, listener wiring, method selector parsing and reporter configuration. + * + *

The deprecated {@code TestNG#configure(org.testng.CommandLineArgs)} keeps a frozen copy of + * this behaviour for the benefit of subclasses such as {@code RemoteTestNG}; both must stay in sync + * until that method is removed. + * + * @since 7.13 + */ +public final class CliConfigurer { + + // Deliberately keyed on TestNG so that the message reads the same as before the CLI extraction. + private static final Logger LOGGER = Logger.getLogger(TestNG.class); + + private static final String BAD_METHOD_SELECTOR = + "Method selector value was not in the format org.example.Selector:4"; + + private CliConfigurer() {} + + /** + * Narrows a class loaded by name without checking it, so that an unsuitable {@code + * -objectfactory} or {@code -testrunfactory} fails where it did before this logic left {@code + * testng-core}: when the instance is created, not here. + */ + @SuppressWarnings("unchecked") + private static Class uncheckedSubclass(Class clazz) { + return (Class) clazz; + } + + /** + * Double checks that the command line parameters are consistent. + * + * @param cli the parsed command line. + * @throws CliParseException when the combination of options cannot select anything to run. + */ + public static void validate(CliOptions cli) { + String testClasses = cli.testClass; + List testNgXml = cli.suiteFiles; + String testJar = cli.testJar; + List methods = cli.commandLineMethods; + + if (testClasses == null + && testJar == null + && (testNgXml == null || testNgXml.isEmpty()) + && (methods == null || methods.isEmpty())) { + throw new CliParseException( + "You need to specify at least one testng.xml, one class or one method"); + } + + String groups = cli.groups; + String excludedGroups = cli.excludedGroups; + + if (testJar == null + && (null != groups || null != excludedGroups) + && testClasses == null + && (testNgXml == null || testNgXml.isEmpty())) { + throw new CliParseException("Groups option should be used with testclass option"); + } + } + + /** + * Applies the parsed command line onto a {@link TestNG} instance. + * + * @param testng the instance to configure. + * @param cli the parsed command line. + */ + public static void configure(TestNG testng, CliOptions cli) { + Objects.requireNonNull( + cli.spiListenersToSkip, CliOptions.LISTENERS_TO_SKIP_VIA_SPI + " must not be null"); + Optional.ofNullable(cli.useGlobalThreadPool).ifPresent(testng::shouldUseGlobalThreadPool); + Optional.ofNullable(cli.shareThreadPoolForDataProviders) + .ifPresent(testng::shareThreadPoolForDataProviders); + // FIXME: the field defaults to FALSE rather than null, so this toggle is switched on for every + // command line run and -propagateDataProviderFailureAsTestFailure cannot be turned off. Kept + // verbatim for parity with the frozen TestNG#configure(CommandLineArgs); fixing it is a + // behaviour change that belongs in its own issue. + Optional.ofNullable(cli.propagateDataProviderFailureAsTestFailure) + .ifPresent(value -> testng.propagateDataProviderFailureAsTestFailure()); + testng.setReportAllDataDrivenTestsAsSkipped(cli.includeAllDataDrivenTestsWhenSkipping); + + Optional.ofNullable(cli.listenerFactory) + .map(ClassHelper::forName) + .filter(ITestNGListenerFactory.class::isAssignableFrom) + .map(it -> it.asSubclass(ITestNGListenerFactory.class)) + .ifPresent(testng::setListenerFactoryClass); + + Optional.ofNullable(cli.generateResultsPerSuite).ifPresent(testng::setGenerateResultsPerSuite); + + Optional.ofNullable(cli.listenerComparator) + .map(ClassHelper::forName) + .filter(ListenerComparator.class::isAssignableFrom) + .map(it -> it.asSubclass(ListenerComparator.class)) + .ifPresent(testng::setListenerComparatorClass); + + if (cli.verbose != null) { + testng.setVerbose(cli.verbose); + } + if (cli.dependencyInjectorFactoryClass != null) { + Class clazz = ClassHelper.forName(cli.dependencyInjectorFactoryClass); + if (clazz != null && IInjectorFactory.class.isAssignableFrom(clazz)) { + testng.setInjectorFactoryClass(clazz.asSubclass(IInjectorFactory.class)); + } + } + Optional.ofNullable(cli.threadPoolFactoryClass) + .map(ClassHelper::forName) + .filter(IExecutorServiceFactory.class::isAssignableFrom) + .map(it -> it.asSubclass(IExecutorServiceFactory.class)) + .ifPresent(testng::setExecutorServiceFactoryClass); + + testng.setOutputDirectory(cli.outputDirectory); + + String testClasses = cli.testClass; + if (null != testClasses) { + String[] strClasses = testClasses.split(","); + List> classes = Lists.newArrayList(); + for (String c : strClasses) { + classes.add(ClassHelper.fileToClass(c)); + } + + testng.setTestClasses(classes.toArray(new Class[0])); + } + + if (cli.testNames != null) { + testng.setTestNames(Arrays.asList(cli.testNames.split(","))); + testng.setIgnoreMissedTestNames(cli.ignoreMissedTestNames); + } + + // Note: can't use a Boolean field here because we are allowing a boolean + // parameter with an arity of 1 ("-usedefaultlisteners false") + if (cli.useDefaultListeners != null) { + testng.setUseDefaultListeners("true".equalsIgnoreCase(cli.useDefaultListeners)); + } + + testng.setGroups(cli.groups); + testng.setExcludedGroups(cli.excludedGroups); + testng.setTestJar(cli.testJar); + testng.setXmlPathInJar(cli.xmlPathInJar); + testng.setSkipFailedInvocationCounts(cli.skipFailedInvocationCounts); + testng.toggleFailureIfAllTestsWereSkipped(cli.failIfAllTestsSkipped); + testng.setListenersToSkipFromBeingWiredInViaServiceLoaders(cli.spiListenersToSkip.split(",")); + + testng.setOverrideIncludedMethods(cli.overrideIncludedMethods); + + if (cli.parallelMode != null) { + testng.setParallel(cli.parallelMode); + } + if (cli.configFailurePolicy != null) { + testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.getValidPolicy(cli.configFailurePolicy)); + } + if (cli.threadCount != null) { + testng.setThreadCount(cli.threadCount); + } + if (cli.dataProviderThreadCount != null) { + testng.setDataProviderThreadCount(cli.dataProviderThreadCount); + } + if (cli.suiteName != null) { + testng.setDefaultSuiteName(cli.suiteName); + } + if (cli.testName != null) { + testng.setDefaultTestName(cli.testName); + } + if (cli.listener != null) { + String sep = ";"; + if (cli.listener.contains(",")) { + sep = ","; + } + String[] strs = Utils.split(cli.listener, sep); + List> classes = Lists.newArrayList(); + + for (String cls : strs) { + Class clazz = ClassHelper.fileToClass(cls); + if (ITestNGListener.class.isAssignableFrom(clazz)) { + classes.add(clazz.asSubclass(ITestNGListener.class)); + } + } + + testng.setListenerClasses(classes); + } + + if (null != cli.methodSelectors) { + String[] strs = Utils.split(cli.methodSelectors, ","); + for (String cls : strs) { + String[] sel = Utils.split(cls, ":"); + try { + if (sel.length == 2) { + testng.addMethodSelector(sel[0], Integer.parseInt(sel[1])); + } else { + LOGGER.error(BAD_METHOD_SELECTOR); + } + } catch (NumberFormatException nfe) { + LOGGER.error(BAD_METHOD_SELECTOR); + } + } + } + + if (cli.objectFactory != null) { + testng.setObjectFactory( + CliConfigurer.uncheckedSubclass( + ClassHelper.fileToClass(cli.objectFactory))); + } + if (cli.testRunnerFactory != null) { + testng.setTestRunnerFactoryClass( + CliConfigurer.uncheckedSubclass( + ClassHelper.fileToClass(cli.testRunnerFactory))); + } + + testng.addReporter(cli.reporter); + + if (!cli.commandLineMethods.isEmpty()) { + testng.setCommandLineMethods(cli.commandLineMethods); + } + + if (cli.suiteFiles != null) { + testng.setTestSuites(cli.suiteFiles); + } + + testng.setSuiteThreadPoolSize(cli.suiteThreadPoolSize); + testng.setRandomizeSuites(cli.randomizeSuites); + testng.alwaysRunListeners(cli.alwaysRunListeners); + } +} diff --git a/testng-cli/src/main/java/org/testng/cli/CliOptions.java b/testng-cli/src/main/java/org/testng/cli/CliOptions.java new file mode 100644 index 000000000..62942d8ae --- /dev/null +++ b/testng-cli/src/main/java/org/testng/cli/CliOptions.java @@ -0,0 +1,197 @@ +package org.testng.cli; + +import java.util.List; +import org.testng.collections.Lists; +import org.testng.xml.XmlSuite; + +/** + * The values a command line parser extracts from {@code argv}, before they are turned into a {@link + * org.testng.TestNG} configuration by {@link CliConfigurer}. + * + *

Fields hold values as close to the raw command line as possible, so that a parser only has to + * fill this object and never has to know about class loading, listener wiring or reporter + * configuration. The single exception is {@link #parallelMode}, which is typed so that parsers + * reject an invalid value instead of silently falling back to {@link XmlSuite.ParallelMode#NONE}. + * + *

The {@code String} constants are the canonical option names. They are duplicated in the + * deprecated {@code org.testng.CommandLineArgs}, which the Maven Surefire integration still reads; + * {@code CliOptionNamesTest} guards the two from drifting apart. + * + * @since 7.13 + */ +public class CliOptions { + + public static final String LOG = "-log"; + public static final String VERBOSE = "-verbose"; + public static final String GROUPS = "-groups"; + public static final String EXCLUDED_GROUPS = "-excludegroups"; + public static final String OUTPUT_DIRECTORY = "-d"; + public static final String MIXED = "-mixed"; + public static final String LISTENER = "-listener"; + public static final String LISTENER_COMPARATOR = "-listenercomparator"; + public static final String METHOD_SELECTORS = "-methodselectors"; + public static final String OBJECT_FACTORY = "-objectfactory"; + public static final String PARALLEL = "-parallel"; + public static final String CONFIG_FAILURE_POLICY = "-configfailurepolicy"; + public static final String THREAD_COUNT = "-threadcount"; + public static final String DATA_PROVIDER_THREAD_COUNT = "-dataproviderthreadcount"; + public static final String SUITE_NAME = "-suitename"; + public static final String TEST_NAME = "-testname"; + public static final String REPORTER = "-reporter"; + public static final String USE_DEFAULT_LISTENERS = "-usedefaultlisteners"; + public static final String SKIP_FAILED_INVOCATION_COUNTS = "-skipfailedinvocationcounts"; + public static final String TEST_CLASS = "-testclass"; + public static final String TEST_NAMES = "-testnames"; + public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames"; + public static final String TEST_JAR = "-testjar"; + public static final String XML_PATH_IN_JAR = "-xmlpathinjar"; + public static final String XML_PATH_IN_JAR_DEFAULT = "testng.xml"; + public static final String TEST_RUNNER_FACTORY = "-testrunfactory"; + public static final String LISTENER_FACTORY = "-listenerfactory"; + public static final String METHODS = "-methods"; + public static final String SUITE_THREAD_POOL_SIZE = "-suitethreadpoolsize"; + public static final Integer SUITE_THREAD_POOL_SIZE_DEFAULT = 1; + public static final String RANDOMIZE_SUITES = "-randomizesuites"; + public static final String ALWAYS_RUN_LISTENERS = "-alwaysrunlisteners"; + public static final String THREAD_POOL_FACTORY_CLASS = "-threadpoolfactoryclass"; + public static final String DEPENDENCY_INJECTOR_FACTORY = "-dependencyinjectorfactory"; + public static final String FAIL_IF_ALL_TESTS_SKIPPED = "-failwheneverythingskipped"; + public static final String LISTENERS_TO_SKIP_VIA_SPI = "-spilistenerstoskip"; + public static final String OVERRIDE_INCLUDED_METHODS = "-overrideincludedmethods"; + public static final String INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING = + "-includeAllDataDrivenTestsWhenSkipping"; + public static final String PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE = + "-propagateDataProviderFailureAsTestFailure"; + public static final String GENERATE_RESULTS_PER_SUITE = "-generateResultsPerSuite"; + public static final String SHARE_THREAD_POOL_FOR_DATA_PROVIDERS = + "-shareThreadPoolForDataProviders"; + public static final String USE_GLOBAL_THREAD_POOL = "-useGlobalThreadPool"; + + /** The XML suite files to run. */ + public List suiteFiles = Lists.newArrayList(); + + /** Level of verbosity. */ + public Integer verbose; + + /** Comma-separated list of group names to be run. */ + public String groups; + + /** Comma-separated list of group names to exclude. */ + public String excludedGroups; + + /** Output directory. */ + public String outputDirectory; + + /** + * List of {@code .class} files or list of class names implementing {@code ITestListener} or + * {@code ISuiteListener}. + */ + public String listener; + + /** An implementation of {@code ListenerComparator} that orders listener execution. */ + public String listenerComparator; + + /** List of {@code .class} files or list of class names implementing {@code IMethodSelector}. */ + public String methodSelectors; + + /** Fully qualified class name that implements {@code org.testng.ITestObjectFactory}. */ + public String objectFactory; + + /** Parallel mode (methods, tests or classes). */ + public XmlSuite.ParallelMode parallelMode; + + /** Configuration failure policy (skip or continue). */ + public String configFailurePolicy; + + /** Number of threads to use when running tests in parallel. */ + public Integer threadCount; + + /** Number of threads to use when running data providers. */ + public Integer dataProviderThreadCount; + + /** Default name of test suite, if not specified in suite definition file or source code. */ + public String suiteName; + + /** Default name of test, if not specified in suite definition file or source code. */ + public String testName; + + /** Extended configuration for custom report listener. */ + public String reporter; + + /** + * Whether to use the default listeners. This is a {@code String} because the option has an arity + * of one ({@code -usedefaultlisteners false}). + */ + public String useDefaultListeners = "true"; + + public Boolean skipFailedInvocationCounts; + + /** The list of test classes. */ + public String testClass; + + /** The list of test names to run. */ + public String testNames; + + /** Ignore missed test names given by {@code -testnames} and continue to run existing tests. */ + public boolean ignoreMissedTestNames = false; + + /** A jar file containing the tests. */ + public String testJar; + + /** The full path to the xml file inside the jar file, only valid with {@code -testjar}. */ + public String xmlPathInJar = XML_PATH_IN_JAR_DEFAULT; + + /** The factory used to create tests. */ + public String testRunnerFactory; + + /** The factory used to create TestNG listeners. */ + public String listenerFactory; + + /** Comma separated list of test methods. */ + public List commandLineMethods = Lists.newArrayList(); + + /** Size of the thread pool to use to run suites. */ + public Integer suiteThreadPoolSize = SUITE_THREAD_POOL_SIZE_DEFAULT; + + /** Whether to run suites in same order as specified in XML or not. */ + public Boolean randomizeSuites = Boolean.FALSE; + + /** Should MethodInvocation Listeners be run even for skipped methods. */ + public Boolean alwaysRunListeners = Boolean.TRUE; + + /** The threadpool executor factory implementation that TestNG should use. */ + public String threadPoolFactoryClass; + + /** The dependency injector factory implementation that TestNG should use. */ + public String dependencyInjectorFactoryClass; + + /** Should TestNG fail execution if all tests were skipped and nothing was run. */ + public Boolean failIfAllTestsSkipped = false; + + /** + * Comma separated fully qualified class names of listeners that should be skipped from being + * wired in via Service Loaders. Never {@code null}: {@link CliConfigurer} splits it eagerly. + */ + public String spiListenersToSkip = ""; + + /** Whether command line method inclusions override the ones declared in the suite XML. */ + public Boolean overrideIncludedMethods = false; + + /** + * Should TestNG report all iterations of a data driven test as individual skips, in case of + * upstream failures. + */ + public Boolean includeAllDataDrivenTestsWhenSkipping = false; + + /** Should TestNG consider failures in Data Providers as test failures. */ + public Boolean propagateDataProviderFailureAsTestFailure = false; + + /** Should TestNG generate results in a sub directory per suite. */ + public Boolean generateResultsPerSuite = false; + + /** Should TestNG use a global shared thread pool for running data providers. */ + public Boolean shareThreadPoolForDataProviders = false; + + /** Should TestNG use a global shared thread pool for regular and data driven tests. */ + public Boolean useGlobalThreadPool = false; +} diff --git a/testng-cli/src/main/java/org/testng/cli/CliParseException.java b/testng-cli/src/main/java/org/testng/cli/CliParseException.java new file mode 100644 index 000000000..aa65abb6c --- /dev/null +++ b/testng-cli/src/main/java/org/testng/cli/CliParseException.java @@ -0,0 +1,22 @@ +package org.testng.cli; + +import org.testng.TestNGException; + +/** + * Raised when a command line cannot be parsed or fails validation. Command line front ends report + * the message and the usage banner, then exit with a failure status. + * + * @since 7.13 + */ +public class CliParseException extends TestNGException { + + private static final long serialVersionUID = 1L; + + public CliParseException(String message) { + super(message); + } + + public CliParseException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/testng-cli/src/test/java/org/testng/cli/CliConfigurerParityTest.java b/testng-cli/src/test/java/org/testng/cli/CliConfigurerParityTest.java new file mode 100644 index 000000000..c13add8aa --- /dev/null +++ b/testng-cli/src/test/java/org/testng/cli/CliConfigurerParityTest.java @@ -0,0 +1,135 @@ +package org.testng.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.testng.CommandLineArgs; +import org.testng.TestNG; +import org.testng.annotations.Test; +import org.testng.xml.XmlSuite; + +/** + * {@link CliConfigurer#configure} and the frozen {@code TestNG#configure(CommandLineArgs)} must + * apply a command line identically for as long as both exist. Nothing else enforces that, so this + * test configures two instances from equivalent inputs and compares the resulting state field by + * field. + */ +public class CliConfigurerParityTest { + + /** + * The only fields that cannot be value-compared: each instance builds its own object and none of + * these types overrides equals. Everything else is either a scalar or an empty collection on both + * sides, and must therefore match. + */ + private static final List IDENTITY_NOT_VALUE = + Arrays.asList( + "m_configuration", + "m_objectFactory", + "exitCodeListener", + "m_annotationTransformer", + "m_defaultAnnoProcessor"); + + @Test + public void configuringViaCliOptionsMatchesTheFrozenCommandLineArgsPath() throws Exception { + TestNG viaCli = new TestNG(); + CliConfigurer.configure(viaCli, populatedCliOptions()); + + FrozenConfigurer viaFrozen = new FrozenConfigurer(); + viaFrozen.apply(populatedCommandLineArgs()); + + Map left = configurationOf(viaCli); + Map right = configurationOf(viaFrozen); + + assertThat(left) + .as("state produced by CliConfigurer vs the frozen TestNG.configure(CommandLineArgs)") + .isEqualTo(right); + // Guard against the comparison silently degenerating to an empty map. + assertThat(left).hasSizeGreaterThan(20); + } + + private static CliOptions populatedCliOptions() { + CliOptions cli = new CliOptions(); + cli.suiteFiles = Arrays.asList("a.xml", "b.xml"); + cli.verbose = 3; + cli.groups = "fast"; + cli.excludedGroups = "slow"; + cli.outputDirectory = "target/parity"; + cli.parallelMode = XmlSuite.ParallelMode.METHODS; + cli.configFailurePolicy = "continue"; + cli.threadCount = 7; + cli.dataProviderThreadCount = 9; + cli.suiteName = "aSuite"; + cli.testName = "aTest"; + cli.useDefaultListeners = "false"; + cli.skipFailedInvocationCounts = Boolean.TRUE; + cli.testNames = "t1,t2"; + cli.ignoreMissedTestNames = true; + cli.testJar = "tests.jar"; + cli.xmlPathInJar = "suites/all.xml"; + cli.commandLineMethods = Arrays.asList("com.acme.A.m1", "com.acme.A.m2"); + cli.suiteThreadPoolSize = 3; + cli.randomizeSuites = Boolean.TRUE; + cli.alwaysRunListeners = Boolean.FALSE; + cli.failIfAllTestsSkipped = Boolean.TRUE; + cli.spiListenersToSkip = "com.acme.Skipped"; + cli.overrideIncludedMethods = Boolean.TRUE; + cli.includeAllDataDrivenTestsWhenSkipping = Boolean.TRUE; + cli.generateResultsPerSuite = Boolean.TRUE; + cli.shareThreadPoolForDataProviders = Boolean.TRUE; + cli.useGlobalThreadPool = Boolean.TRUE; + return cli; + } + + /** + * Copies the populated options into the deprecated bag by field name. Reflective on purpose: a + * hand-written copy that silently misses a field would weaken the comparison in the permissive + * direction. {@code CliOptionNamesTest} already pins that the two classes agree. + */ + @SuppressWarnings("deprecation") + private static CommandLineArgs populatedCommandLineArgs() throws IllegalAccessException { + CommandLineArgs args = new CommandLineArgs(); + CliOptions cli = populatedCliOptions(); + for (Field source : CliOptions.class.getDeclaredFields()) { + if (java.lang.reflect.Modifier.isStatic(source.getModifiers()) || source.isSynthetic()) { + continue; + } + Field target; + try { + target = CommandLineArgs.class.getDeclaredField(source.getName()); + } catch (NoSuchFieldException e) { + throw new AssertionError("CommandLineArgs has no " + source.getName(), e); + } + target.set(args, source.get(cli)); + } + return args; + } + + /** Reads every comparable instance field of {@link TestNG}, including inherited ones. */ + private static Map configurationOf(TestNG testng) throws IllegalAccessException { + Map state = new LinkedHashMap<>(); + for (Field field : TestNG.class.getDeclaredFields()) { + if (java.lang.reflect.Modifier.isStatic(field.getModifiers()) + || field.isSynthetic() + || IDENTITY_NOT_VALUE.contains(field.getName())) { + continue; + } + field.setAccessible(true); + Object value = field.get(testng); + state.put( + field.getName(), value instanceof Object[] ? Arrays.asList((Object[]) value) : value); + } + return state; + } + + /** Gives access to the {@code protected} frozen configuration path kept on {@link TestNG}. */ + private static final class FrozenConfigurer extends TestNG { + @SuppressWarnings("deprecation") + void apply(CommandLineArgs args) { + configure(args); + } + } +} diff --git a/testng-cli/src/test/java/org/testng/cli/CliConfigurerValidateTest.java b/testng-cli/src/test/java/org/testng/cli/CliConfigurerValidateTest.java new file mode 100644 index 000000000..86c6ba4f5 --- /dev/null +++ b/testng-cli/src/test/java/org/testng/cli/CliConfigurerValidateTest.java @@ -0,0 +1,109 @@ +package org.testng.cli; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +import java.util.Arrays; +import java.util.Collections; +import org.testng.CommandLineArgs; +import org.testng.TestNG; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class CliConfigurerValidateTest { + + @Test + public void nothingToRunIsRejected() { + assertThatThrownBy(() -> CliConfigurer.validate(new CliOptions())) + .isInstanceOf(CliParseException.class) + .hasMessageContaining( + "You need to specify at least one testng.xml, one class or one method"); + } + + @Test + public void groupsWithoutTestClassIsRejected() { + CliOptions cli = new CliOptions(); + cli.commandLineMethods = Collections.singletonList("com.acme.Sample.aMethod"); + cli.groups = "fast"; + + assertThatThrownBy(() -> CliConfigurer.validate(cli)) + .isInstanceOf(CliParseException.class) + .hasMessageContaining("Groups option should be used with testclass option"); + } + + @Test + public void groupsWithTestClassIsAccepted() { + CliOptions cli = new CliOptions(); + cli.testClass = "com.acme.Sample"; + cli.groups = "fast"; + + assertThatCode(() -> CliConfigurer.validate(cli)).doesNotThrowAnyException(); + } + + @Test + public void suiteFilesAloneAreAccepted() { + CliOptions cli = new CliOptions(); + cli.suiteFiles = Arrays.asList("testng.xml"); + + assertThatCode(() -> CliConfigurer.validate(cli)).doesNotThrowAnyException(); + } + + @DataProvider + public Object[][] commandLines() { + return new Object[][] { + {null, null, null, null, Collections.emptyList()}, + {"com.acme.Sample", null, null, null, Collections.emptyList()}, + {null, "tests.jar", null, null, Collections.emptyList()}, + {null, null, null, null, Collections.singletonList("com.acme.Sample.aMethod")}, + {null, null, "fast", null, Collections.singletonList("com.acme.Sample.aMethod")}, + {null, null, null, "slow", Collections.singletonList("com.acme.Sample.aMethod")}, + {"com.acme.Sample", null, "fast", null, Collections.emptyList()}, + {null, "tests.jar", "fast", null, Collections.emptyList()}, + }; + } + + /** + * The frozen {@code TestNG.validateCommandLineParameters} must keep accepting and rejecting + * exactly what {@link CliConfigurer#validate} does, for as long as both exist. + */ + @Test(dataProvider = "commandLines") + public void validationMatchesTheDeprecatedTestNgEntryPoint( + String testClass, + String testJar, + String groups, + String excludedGroups, + java.util.List methods) { + CliOptions cli = new CliOptions(); + cli.testClass = testClass; + cli.testJar = testJar; + cli.groups = groups; + cli.excludedGroups = excludedGroups; + cli.commandLineMethods = methods; + + @SuppressWarnings("deprecation") + CommandLineArgs args = new CommandLineArgs(); + args.testClass = testClass; + args.testJar = testJar; + args.groups = groups; + args.excludedGroups = excludedGroups; + args.commandLineMethods = methods; + + Throwable fromCli = catchThrowable(() -> CliConfigurer.validate(cli)); + Throwable fromTestNg = catchThrowable(() -> LegacyValidator.validate(args)); + + assertThat(fromCli == null).isEqualTo(fromTestNg == null); + if (fromCli != null) { + assertThat(fromCli.getMessage()).isEqualTo(fromTestNg.getMessage()); + } + } + + /** Gives access to the {@code protected static} validation kept on {@link TestNG}. */ + private static final class LegacyValidator extends TestNG { + @SuppressWarnings("deprecation") + static void validate(CommandLineArgs args) { + validateCommandLineParameters(args); + } + } +} diff --git a/testng-cli/src/test/java/org/testng/cli/CliOptionNamesTest.java b/testng-cli/src/test/java/org/testng/cli/CliOptionNamesTest.java new file mode 100644 index 000000000..25363d7cc --- /dev/null +++ b/testng-cli/src/test/java/org/testng/cli/CliOptionNamesTest.java @@ -0,0 +1,41 @@ +package org.testng.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.Map; +import java.util.TreeMap; +import org.testng.CommandLineArgs; +import org.testng.annotations.Test; + +/** + * {@link CliOptions} owns the canonical option names, but the deprecated {@link CommandLineArgs} + * still declares its own copy for the Maven Surefire integration. They must not drift apart while + * both exist. + */ +public class CliOptionNamesTest { + + @Test + public void optionNamesMatchTheDeprecatedCommandLineArgs() { + assertThat(constantsOf(CliOptions.class)) + .as("option constants of %s", CliOptions.class.getName()) + .isEqualTo(constantsOf(CommandLineArgs.class)); + } + + private static Map constantsOf(Class type) { + Map constants = new TreeMap<>(); + for (Field field : type.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (!Modifier.isStatic(modifiers) || !Modifier.isPublic(modifiers)) { + continue; + } + try { + constants.put(field.getName(), field.get(null)); + } catch (IllegalAccessException e) { + throw new AssertionError("Cannot read " + type.getName() + "." + field.getName(), e); + } + } + return constants; + } +} diff --git a/testng-cli/testng-cli-build.gradle.kts b/testng-cli/testng-cli-build.gradle.kts new file mode 100644 index 000000000..e1740beb8 --- /dev/null +++ b/testng-cli/testng-cli-build.gradle.kts @@ -0,0 +1,9 @@ +plugins { + id("testng.java-library") +} + +description = "Command line contract and shared configuration logic for TestNG" + +dependencies { + api(projects.testngCore) +} diff --git a/testng-core/src/main/java/org/testng/CliRunners.java b/testng-core/src/main/java/org/testng/CliRunners.java new file mode 100644 index 000000000..37a5052be --- /dev/null +++ b/testng-core/src/main/java/org/testng/CliRunners.java @@ -0,0 +1,113 @@ +package org.testng; + +import java.util.Iterator; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import org.testng.log4testng.Logger; + +/** + * Resolves the {@link ITestNGCliRunner} service. + * + *

Note: this class is not part of the public API and is meant for internal usage only. + */ +final class CliRunners { + + private static final Logger LOGGER = Logger.getLogger(CliRunners.class); + + private static final String MISSING = + "TestNG command line support is not available: no implementation of " + + ITestNGCliRunner.class.getName() + + " was found on the classpath.\n" + + "The org.testng:testng jar bundles one. If TestNG was repackaged, or is consumed as " + + "individual modules, make sure a runner and its META-INF/services entry are present " + + "(along with its parsing library), or drive TestNG through the org.testng.TestNG Java " + + "API instead."; + + private static volatile ITestNGCliRunner cached; + + /** + * Why the last lookup came back empty. A provider that is present but fails to load reports the + * very same "nothing found" outcome as a provider that is simply absent, so the cause has to be + * carried along or the diagnostic tells people to install what they already installed. + */ + private static volatile Throwable lastFailure; + + private CliRunners() {} + + /** @return the installed runner, or {@code null} when none is available. */ + static ITestNGCliRunner find() { + ITestNGCliRunner local = cached; + if (local != null) { + return local; + } + // The classloader that owns the SPI. On a plain classpath this is the application classloader. + // Inside the merged testng.jar OSGi bundle this is the bundle classloader, which also owns the + // provider class and its META-INF/services entry, so no SPI-Fly weaving is required. + lastFailure = null; + ClassLoader owner = ITestNGCliRunner.class.getClassLoader(); + local = load(owner); + if (local == null) { + // Fallback for parent/child deployments where only the child sees the provider. + ClassLoader context = Thread.currentThread().getContextClassLoader(); + if (context != null && context != owner) { + local = load(context); + } + } + if (local != null) { + // A null result is deliberately not cached: the provider may show up on a later lookup, and + // caching it would also let a provider-less lookup clobber a concurrent successful one. + cached = local; + } + return local; + } + + /** + * @return the installed runner. + * @throws TestNGException when no runner is available. + */ + static ITestNGCliRunner required() { + ITestNGCliRunner runner = find(); + if (runner == null) { + Throwable cause = lastFailure; + throw cause == null ? new TestNGException(MISSING) : new TestNGException(MISSING, cause); + } + return runner; + } + + private static ITestNGCliRunner load(ClassLoader loader) { + try { + Iterator it = ServiceLoader.load(ITestNGCliRunner.class, loader).iterator(); + if (!it.hasNext()) { + return null; + } + ITestNGCliRunner runner = it.next(); + warnIfAnotherProviderFollows(it, runner); + return runner; + } catch (ServiceConfigurationError | RuntimeException e) { + lastFailure = e; + return null; + } + } + + /** + * Peeks at the next provider only to report an ambiguity. A failure here says nothing about the + * runner already in hand, so it must not cost us that runner. + */ + private static void warnIfAnotherProviderFollows( + Iterator it, ITestNGCliRunner chosen) { + boolean ambiguous; + try { + ambiguous = it.hasNext(); + } catch (ServiceConfigurationError | RuntimeException ignored) { + return; + } + if (ambiguous) { + LOGGER.warn( + "Several " + + ITestNGCliRunner.class.getName() + + " implementations are on the classpath. Using " + + chosen.getClass().getName() + + ". Ordering is defined by the classloader and is not stable."); + } + } +} diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java index 3bdf7feb5..0bc47f053 100644 --- a/testng-core/src/main/java/org/testng/CommandLineArgs.java +++ b/testng-core/src/main/java/org/testng/CommandLineArgs.java @@ -1,288 +1,234 @@ package org.testng; -import com.beust.jcommander.Parameter; import java.util.ArrayList; import java.util.List; import org.testng.collections.Lists; import org.testng.xml.XmlSuite; +/** + * The values the TestNG command line used to be parsed into. + * + *

Since 7.13 the command line front end lives in the {@code testng-cli} and {@code + * testng-jcommander} modules, so that {@code testng-core} no longer depends on a command line + * parsing library. This class is kept, without its JCommander annotations, only so that {@link + * TestNG#configure(CommandLineArgs)}, {@link TestNG#configure(java.util.Map)} and subclasses such + * as {@code RemoteTestNG} keep working. It is no longer populated by TestNG itself. + * + * @deprecated since 7.13. Use {@code org.testng.cli.CliOptions} from the {@code testng-cli} module. + * Scheduled for removal in 8.0. + */ +@Deprecated public class CommandLineArgs { - @Parameter(description = "The XML suite files to run") + /** The XML suite files to run. */ public List suiteFiles = Lists.newArrayList(); public static final String LOG = "-log"; public static final String VERBOSE = "-verbose"; - @Parameter( - names = {LOG, VERBOSE}, - description = "Level of verbosity") + /** Level of verbosity. */ public Integer verbose; public static final String GROUPS = "-groups"; - @Parameter(names = GROUPS, description = "Comma-separated list of group names to be run") + /** Comma-separated list of group names to be run. */ public String groups; public static final String EXCLUDED_GROUPS = "-excludegroups"; - @Parameter( - names = EXCLUDED_GROUPS, - description = "Comma-separated list of group names to " + " exclude") + /** Comma-separated list of group names to exclude. */ public String excludedGroups; public static final String OUTPUT_DIRECTORY = "-d"; - @Parameter(names = OUTPUT_DIRECTORY, description = "Output directory") + /** Output directory. */ public String outputDirectory; public static final String MIXED = "-mixed"; - @Parameter( - names = MIXED, - description = - "No-op since JUnit execution support was removed in 7.10.0." - + " Kept for command line backward compatibility.") + /** + * No-op since JUnit execution support was removed in 7.10.0. Kept for command line backward + * compatibility. + */ public Boolean mixed = Boolean.FALSE; public static final String LISTENER = "-listener"; - @Parameter( - names = LISTENER, - description = - "List of .class files or list of class names" - + " implementing ITestListener or ISuiteListener") + /** List of .class files or list of class names implementing ITestListener or ISuiteListener. */ public String listener; public static final String LISTENER_COMPARATOR = "-listenercomparator"; - @Parameter( - names = LISTENER_COMPARATOR, - description = - "An implementation of ListenerComparator that will be used by TestNG to determine order of execution for listeners") + /** An implementation of ListenerComparator that determines order of execution for listeners. */ public String listenerComparator; public static final String METHOD_SELECTORS = "-methodselectors"; - @Parameter( - names = METHOD_SELECTORS, - description = "List of .class files or list of class " + "names implementing IMethodSelector") + /** List of .class files or list of class names implementing IMethodSelector. */ public String methodSelectors; public static final String OBJECT_FACTORY = "-objectfactory"; - @Parameter( - names = OBJECT_FACTORY, - description = - "Fully qualified class name that implements org.testng.ITestObjectFactory which can be used to create test class and listener instances.") + /** Fully qualified class name that implements org.testng.ITestObjectFactory. */ public String objectFactory; public static final String PARALLEL = "-parallel"; - @Parameter(names = PARALLEL, description = "Parallel mode (methods, tests or classes)") + /** Parallel mode (methods, tests or classes). */ public XmlSuite.ParallelMode parallelMode; public static final String CONFIG_FAILURE_POLICY = "-configfailurepolicy"; - @Parameter( - names = CONFIG_FAILURE_POLICY, - description = "Configuration failure policy (skip or continue)") + /** Configuration failure policy (skip or continue). */ public String configFailurePolicy; public static final String THREAD_COUNT = "-threadcount"; - @Parameter( - names = THREAD_COUNT, - description = "Number of threads to use when running tests " + "in parallel") + /** Number of threads to use when running tests in parallel. */ public Integer threadCount; public static final String DATA_PROVIDER_THREAD_COUNT = "-dataproviderthreadcount"; - @Parameter( - names = DATA_PROVIDER_THREAD_COUNT, - description = "Number of threads to use when " + "running data providers") + /** Number of threads to use when running data providers. */ public Integer dataProviderThreadCount; public static final String SUITE_NAME = "-suitename"; - @Parameter( - names = SUITE_NAME, - description = - "Default name of test suite, if not specified " - + "in suite definition file or source code") + /** Default name of test suite, if not specified in suite definition file or source code. */ public String suiteName; public static final String TEST_NAME = "-testname"; - @Parameter( - names = TEST_NAME, - description = - "Default name of test, if not specified in suite" + "definition file or source code") + /** Default name of test, if not specified in suite definition file or source code. */ public String testName; public static final String REPORTER = "-reporter"; - @Parameter(names = REPORTER, description = "Extended configuration for custom report listener") + /** Extended configuration for custom report listener. */ public String reporter; public static final String USE_DEFAULT_LISTENERS = "-usedefaultlisteners"; - @Parameter(names = USE_DEFAULT_LISTENERS, description = "Whether to use the default listeners") + /** Whether to use the default listeners. */ public String useDefaultListeners = "true"; public static final String SKIP_FAILED_INVOCATION_COUNTS = "-skipfailedinvocationcounts"; - @Parameter(names = SKIP_FAILED_INVOCATION_COUNTS, hidden = true) public Boolean skipFailedInvocationCounts; public static final String TEST_CLASS = "-testclass"; - @Parameter(names = TEST_CLASS, description = "The list of test classes") + /** The list of test classes. */ public String testClass; public static final String TEST_NAMES = "-testnames"; - @Parameter(names = TEST_NAMES, description = "The list of test names to run") + /** The list of test names to run. */ public String testNames; public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames"; - @Parameter( - names = IGNORE_MISSED_TEST_NAMES, - description = - "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.") + /** Ignore missed test names given by '-testnames' and continue to run existing tests, if any. */ public boolean ignoreMissedTestNames = false; public static final String TEST_JAR = "-testjar"; - @Parameter(names = TEST_JAR, description = "A jar file containing the tests") + /** A jar file containing the tests. */ public String testJar; public static final String XML_PATH_IN_JAR = "-xmlpathinjar"; public static final String XML_PATH_IN_JAR_DEFAULT = "testng.xml"; - @Parameter( - names = XML_PATH_IN_JAR, - description = - "The full path to the xml file inside the jar file (only valid if -testjar was specified)") + /** The full path to the xml file inside the jar file, only valid with -testjar. */ public String xmlPathInJar = XML_PATH_IN_JAR_DEFAULT; public static final String TEST_RUNNER_FACTORY = "-testrunfactory"; - @Parameter( - names = {TEST_RUNNER_FACTORY, "-testRunFactory"}, - description = "The factory used to create tests") + /** The factory used to create tests. */ public String testRunnerFactory; public static final String LISTENER_FACTORY = "-listenerfactory"; - @Parameter(names = LISTENER_FACTORY, description = "The factory used to create TestNG listeners") + /** The factory used to create TestNG listeners. */ public String listenerFactory; public static final String METHODS = "-methods"; - @Parameter(names = METHODS, description = "Comma separated of test methods") + /** Comma separated list of test methods. */ public List commandLineMethods = new ArrayList<>(); public static final String SUITE_THREAD_POOL_SIZE = "-suitethreadpoolsize"; public static final Integer SUITE_THREAD_POOL_SIZE_DEFAULT = 1; - @Parameter( - names = SUITE_THREAD_POOL_SIZE, - description = "Size of the thread pool to use to run suites") + /** Size of the thread pool to use to run suites. */ public Integer suiteThreadPoolSize = SUITE_THREAD_POOL_SIZE_DEFAULT; public static final String RANDOMIZE_SUITES = "-randomizesuites"; - @Parameter( - names = RANDOMIZE_SUITES, - hidden = true, - description = "Whether to run suites in same order as specified in XML or not") + /** Whether to run suites in same order as specified in XML or not. */ public Boolean randomizeSuites = Boolean.FALSE; public static final String ALWAYS_RUN_LISTENERS = "-alwaysrunlisteners"; - @Parameter( - names = ALWAYS_RUN_LISTENERS, - description = "Should MethodInvocation Listeners be run even for skipped methods") + /** Should MethodInvocation Listeners be run even for skipped methods. */ public Boolean alwaysRunListeners = Boolean.TRUE; public static final String THREAD_POOL_FACTORY_CLASS = "-threadpoolfactoryclass"; - @Parameter( - names = THREAD_POOL_FACTORY_CLASS, - description = "The threadpool executor factory implementation that TestNG should use.") + /** The threadpool executor factory implementation that TestNG should use. */ public String threadPoolFactoryClass; public static final String DEPENDENCY_INJECTOR_FACTORY = "-dependencyinjectorfactory"; - @Parameter( - names = DEPENDENCY_INJECTOR_FACTORY, - description = "The dependency injector factory implementation that TestNG should use.") + /** The dependency injector factory implementation that TestNG should use. */ public String dependencyInjectorFactoryClass; public static final String FAIL_IF_ALL_TESTS_SKIPPED = "-failwheneverythingskipped"; - @Parameter( - names = FAIL_IF_ALL_TESTS_SKIPPED, - description = "Should TestNG fail execution if all tests were skipped and nothing was run.") + /** Should TestNG fail execution if all tests were skipped and nothing was run. */ public Boolean failIfAllTestsSkipped = false; public static final String LISTENERS_TO_SKIP_VIA_SPI = "-spilistenerstoskip"; - @Parameter( - names = LISTENERS_TO_SKIP_VIA_SPI, - description = - "Comma separated fully qualified class names of listeners that should be skipped from being wired in via Service Loaders.") + /** + * Comma separated fully qualified class names of listeners that should be skipped from being + * wired in via Service Loaders. + */ public String spiListenersToSkip = ""; public static final String OVERRIDE_INCLUDED_METHODS = "-overrideincludedmethods"; - @Parameter( - names = OVERRIDE_INCLUDED_METHODS, - description = - "Comma separated fully qualified class names of listeners that should be skipped from being wired in via Service Loaders.") + /** Whether command line method inclusions override the ones declared in the suite XML. */ public Boolean overrideIncludedMethods = false; public static final String INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING = "-includeAllDataDrivenTestsWhenSkipping"; - @Parameter( - names = INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING, - description = - "Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.") + /** + * Should TestNG report all iterations of a data driven test as individual skips, in-case of + * upstream failures. + */ public Boolean includeAllDataDrivenTestsWhenSkipping = false; public static final String PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE = "-propagateDataProviderFailureAsTestFailure"; - @Parameter( - names = PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE, - description = "Should TestNG consider failures in Data Providers as test failures.") + /** Should TestNG consider failures in Data Providers as test failures. */ public Boolean propagateDataProviderFailureAsTestFailure = false; public static final String GENERATE_RESULTS_PER_SUITE = "-generateResultsPerSuite"; - @Parameter( - names = GENERATE_RESULTS_PER_SUITE, - description = - "Should TestNG generate results on a per suite basis by creating a sub directory for each suite and dumping results into it.") + /** Should TestNG generate results in a sub directory per suite. */ public Boolean generateResultsPerSuite = false; public static final String SHARE_THREAD_POOL_FOR_DATA_PROVIDERS = "-shareThreadPoolForDataProviders"; - @Parameter( - names = SHARE_THREAD_POOL_FOR_DATA_PROVIDERS, - description = - "Should TestNG use a global Shared ThreadPool (At suite level) for running data providers.") + /** Should TestNG use a global shared thread pool for running data providers. */ public Boolean shareThreadPoolForDataProviders = false; public static final String USE_GLOBAL_THREAD_POOL = "-useGlobalThreadPool"; - @Parameter( - names = USE_GLOBAL_THREAD_POOL, - description = - "Should TestNG use a global Shared ThreadPool (At suite level) for running regular and data driven tests.") + /** Should TestNG use a global shared thread pool for regular and data driven tests. */ public Boolean useGlobalThreadPool = false; } diff --git a/testng-core/src/main/java/org/testng/ITestNGCliRunner.java b/testng-core/src/main/java/org/testng/ITestNGCliRunner.java new file mode 100644 index 000000000..1ea21c484 --- /dev/null +++ b/testng-core/src/main/java/org/testng/ITestNGCliRunner.java @@ -0,0 +1,39 @@ +package org.testng; + +/** + * Service provider interface backing {@link TestNG#main(String[])}. Implementations are discovered + * with {@link java.util.ServiceLoader}, which keeps the command line parsing library out of {@code + * testng-core}. + * + *

The reference implementation lives in the {@code testng-jcommander} module and is bundled + * inside the {@code org.testng:testng} jar. + * + *

Implementations must be embeddable: they never terminate the JVM and never write a usage + * banner of their own accord. Deciding what a bad command line costs the process belongs to the + * entry point, which is {@link TestNG#main(String[])}. + * + * @since 7.13 + */ +public interface ITestNGCliRunner { + + /** + * Parses the command line, configures a fresh {@link TestNG} instance and runs it. + * + *

A run that fails is not an error here: the returned instance carries the outcome in + * {@link TestNG#getStatus()}. Only a command line that cannot be honoured is signalled as an + * exception. + * + * @param argv the TestNG command line parameters + * @param listener an optional listener to register before the run, may be {@code null} + * @return the {@link TestNG} instance that was run + * @throws TestNGException when {@code argv} cannot be parsed or does not select anything to run. + * The message is meant to be shown to the user as is. + */ + TestNG run(String[] argv, ITestListener listener); + + /** + * Prints the command line usage banner. Callers treat a throw as "no banner available" and fall + * back to a terse built-in one. + */ + void usage(); +} diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 4bfd9c945..c872ae5d1 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -5,8 +5,6 @@ import static org.testng.internal.Utils.isStringEmpty; import static org.testng.internal.Utils.isStringNotEmpty; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.ParameterException; import java.io.File; import java.io.IOException; import java.net.URLClassLoader; @@ -124,8 +122,6 @@ public class TestNG { private static TestNG m_instance; - private static JCommander m_jCommander; - private List m_commandLineMethods; protected List m_suites = Lists.newArrayList(); private List m_cmdlineSuites; @@ -276,6 +272,15 @@ public void setListenerComparator(ListenerComparator listenerComparator) { this.m_configuration.setListenerComparator(listenerComparator); } + /** + * @param listenerComparatorClass a {@link ListenerComparator} implementation, instantiated with + * the object factory currently in use. + */ + public void setListenerComparatorClass( + Class listenerComparatorClass) { + setListenerComparator(m_objectFactory.newInstance(listenerComparatorClass)); + } + public ListenerComparator getListenerComparator() { return m_configuration.getListenerComparator(); } @@ -428,7 +433,10 @@ public void initializeSuitesAndJarFile() { /** @param threadCount Define the number of threads in the thread pool. */ public void setThreadCount(int threadCount) { if (threadCount < 1) { - exitWithError("Cannot use a threadCount parameter less than 1; 1 > " + threadCount); + // Reported rather than fatal: killing the JVM from a setter breaks every embedder, and + // TestNG#main is the one place allowed to turn a bad value into an exit code. + throw new TestNGException( + "Cannot use a threadCount parameter less than 1; 1 > " + threadCount); } m_threadCount = threadCount; @@ -663,7 +671,7 @@ public void setGroups(String groups) { m_includedGroups = Utils.split(groups, ","); } - private void setTestRunnerFactoryClass( + public void setTestRunnerFactoryClass( Class testRunnerFactoryClass) { setTestRunnerFactory(m_objectFactory.newInstance(testRunnerFactoryClass)); } @@ -846,10 +854,27 @@ public void setExecutorServiceFactory(IExecutorServiceFactory factory) { Objects.requireNonNull(factory, "ExecutorServiceFactory cannot be null")); } + /** + * @param factoryClass an {@link IExecutorServiceFactory} implementation, instantiated with the + * object factory currently in use. + */ + public void setExecutorServiceFactoryClass( + Class factoryClass) { + setExecutorServiceFactory(m_objectFactory.newInstance(factoryClass)); + } + public void setListenerFactory(ITestNGListenerFactory factory) { this.m_configuration.setListenerFactory(factory); } + /** + * @param factoryClass an {@link ITestNGListenerFactory} implementation, instantiated with the + * object factory currently in use. + */ + public void setListenerFactoryClass(Class factoryClass) { + setListenerFactory(m_objectFactory.newInstance(factoryClass)); + } + public void setGenerateResultsPerSuite(boolean generateResultsPerSuite) { this.m_generateResultsPerSuite = generateResultsPerSuite; } @@ -1093,7 +1118,6 @@ public void run() { } m_instance = null; - m_jCommander = null; } /** @@ -1140,10 +1164,25 @@ private void runExecutionListeners(boolean start) { } private static void usage() { - if (m_jCommander == null) { - m_jCommander = new JCommander(new CommandLineArgs()); + // find() rather than required(): printing usage must never be the thing that fails. + ITestNGCliRunner runner = CliRunners.find(); + if (runner != null) { + try { + runner.usage(); + return; + } catch (RuntimeException ignored) { + // A third party runner must not break the callers of usage(), which include plain Java API + // paths such as runSuitesLocally(). Fall through to the built-in banner. + } } - m_jCommander.usage(); + // Written to stdout, like the banner a command line runner produces. + System.out.println( + "Usage: java " + + TestNG.class.getName() + + " [options] ...\n" + + "Detailed command line help requires an " + + ITestNGCliRunner.class.getName() + + " implementation on the classpath; the org.testng:testng jar bundles one."); } private void generateReports(List suiteRunners) { @@ -1385,10 +1424,25 @@ protected IConfiguration getConfiguration() { /** * The TestNG entry point for command line execution. * + *

Kept working so that {@code java -cp testng.jar org.testng.TestNG suite.xml} keeps behaving + * as before: it hands {@code argv} over to the {@link ITestNGCliRunner} found on the classpath. + * * @param argv the TestNG command line parameters. + * @deprecated since 7.13. The command line front end now lives in its own modules; invoke {@code + * org.testng.cli.jcommander.JCommanderCliRunner} (or any other {@link ITestNGCliRunner}) + * directly, or drive TestNG through its Java API. Scheduled for removal in 8.0. */ + @Deprecated public static void main(String[] argv) { - TestNG testng = privateMain(argv, null); + TestNG testng; + try { + testng = privateMain(argv, null); + } catch (TestNGException ex) { + // Typically no ITestNGCliRunner on the classpath. Report it the way any other command line + // error is reported rather than as an uncaught stack trace. + exitWithError(ex.getMessage()); + return; + } System.exit(testng.getStatus()); } @@ -1398,50 +1452,27 @@ public static void main(String[] argv) { * @param argv The param arguments * @param listener The listener * @return The TestNG instance + * @throws TestNGException when no {@link ITestNGCliRunner} implementation is on the classpath, or + * when {@code argv} cannot be honoured. Unlike before 7.13, an unusable command line no + * longer terminates the JVM here; only {@link #main(String[])} does that. + * @deprecated since 7.13. Use {@link ITestNGCliRunner#run(String[], ITestListener)} on the runner + * of your choice. Scheduled for removal in 8.0. */ + @Deprecated public static TestNG privateMain(String[] argv, ITestListener listener) { - TestNG result = new TestNG(); - - if (null != listener) { - result.addListener(listener); - } - - // - // Parse the arguments - // - try { - CommandLineArgs cla = new CommandLineArgs(); - - m_jCommander = new JCommander(cla); - m_jCommander.parse(argv); - validateCommandLineParameters(cla); - result.configure(cla); - } catch (ParameterException ex) { - exitWithError(ex.getMessage()); - } - - // - // Run - // - try { - result.run(); - } catch (TestNGException ex) { - if (TestRunner.getVerbose() > 1) { - ex.printStackTrace(System.out); - } else { - error(ex.getMessage()); - } - result.exitCode = ExitCode.newExitCodeRepresentingFailure(); - } - - return result; + return CliRunners.required().run(argv, listener); } /** * Configure the TestNG instance based on the command line parameters. * * @param cla The command line parameters + * @deprecated since 7.13. The command line front end lives in the {@code testng-cli} module; use + * {@code org.testng.cli.CliConfigurer#configure(TestNG, org.testng.cli.CliOptions)}. This + * method is frozen for the benefit of subclasses such as {@code RemoteTestNG} and will be + * removed in 8.0. */ + @Deprecated protected void configure(CommandLineArgs cla) { Optional.ofNullable(cla.useGlobalThreadPool).ifPresent(this::shouldUseGlobalThreadPool); Optional.ofNullable(cla.shareThreadPoolForDataProviders) @@ -1598,10 +1629,48 @@ protected void configure(CommandLineArgs cla) { alwaysRunListeners(cla.alwaysRunListeners); } - private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) { + public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) { m_ignoreMissedTestNames = ignoreMissedTestNames; } + /** + * Restricts this run to the given fully qualified method names. + * + *

A {@code null} or empty list clears the restriction. An empty list must not be stored as is: + * {@code initializeCommandLineSuites} branches on the field being non-null, so an empty one would + * build a suite with no class and silently suppress {@link #setTestClasses(Class[])}. + * + * @param methods a list of fully qualified method names, as accepted by the {@code -methods} + * command line option. + */ + public void setCommandLineMethods(List methods) { + m_commandLineMethods = + methods == null || methods.isEmpty() ? null : Lists.newArrayList(methods); + } + + public void setOverrideIncludedMethods(boolean overrideIncludedMethods) { + m_configuration.setOverrideIncludedMethods(overrideIncludedMethods); + } + + /** + * Reports a failure raised by {@link #run()} itself and records it, so that {@link #getStatus()} + * says the run failed even though no test result does. + * + *

A front end that would rather report a broken run as an exit status than let the exception + * propagate catches it and calls this. How much of the failure is printed depends on the current + * verbosity, which is why this lives here rather than in the caller. + * + * @param cause the failure raised by the run. + */ + public void reportRunFailure(TestNGException cause) { + if (TestRunner.getVerbose() > 1) { + cause.printStackTrace(System.out); + } else { + error(cause.getMessage()); + } + this.exitCode = ExitCode.newExitCodeRepresentingFailure(); + } + public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) { m_suiteThreadPoolSize = suiteThreadPoolSize; } @@ -1786,6 +1855,24 @@ public void setSkipFailedInvocationCounts(Boolean skip) { m_skipFailedInvocationCounts = skip; } + /** + * Adds a reporter described the way the {@code -reporter} command line option does, for instance + * {@code com.acme.MyReporter:fileName=out.html}. + * + *

The class is resolved and instantiated synchronously. A {@code null} or empty value is + * silently ignored; a class that cannot be found is reported as a warning and skipped; a class + * that is found but does not implement {@link IReporter} raises a {@link TestNGException}. + * + * @param reporterConfigString the serialized reporter configuration. + * @throws TestNGException if the named class is not an {@link IReporter}. + */ + public void addReporter(String reporterConfigString) { + ReporterConfig reporterConfig = ReporterConfig.deserialize(reporterConfigString); + if (reporterConfig != null) { + addReporter(reporterConfig); + } + } + private void addReporter(ReporterConfig reporterConfig) { IReporter instance = newReporterInstance(reporterConfig); if (instance != null) { @@ -1817,7 +1904,11 @@ private IReporter newReporterInstance(ReporterConfig config) { * Double check that the command line parameters are valid. * * @param args The command line to check + * @deprecated since 7.13. Use {@code org.testng.cli.CliConfigurer#validate} from the {@code + * testng-cli} module. Note that failures are now reported as {@link TestNGException} instead + * of JCommander's {@code ParameterException}. */ + @Deprecated protected static void validateCommandLineParameters(CommandLineArgs args) { String testClasses = args.testClass; List testNgXml = args.suiteFiles; @@ -1828,7 +1919,7 @@ protected static void validateCommandLineParameters(CommandLineArgs args) { && testJar == null && (testNgXml == null || testNgXml.isEmpty()) && (methods == null || methods.isEmpty())) { - throw new ParameterException( + throw new TestNGException( "You need to specify at least one testng.xml, one class" + " or one method"); } @@ -1839,7 +1930,7 @@ protected static void validateCommandLineParameters(CommandLineArgs args) { && (null != groups || null != excludedGroups) && testClasses == null && (testNgXml == null || testNgXml.isEmpty())) { - throw new ParameterException("Groups option should be used with testclass option"); + throw new TestNGException("Groups option should be used with testclass option"); } } @@ -1859,7 +1950,9 @@ public boolean hasSkip() { } static void exitWithError(String msg) { - System.err.println(msg); + // Trimmed because TestNGException prefixes every message with a newline, which would show up + // as a stray blank line ahead of the error. + System.err.println(msg == null ? "" : msg.trim()); usage(); System.exit(1); } @@ -2015,4 +2108,12 @@ public List getServiceLoaderListeners() { public void setInjectorFactory(IInjectorFactory factory) { this.m_configuration.setInjectorFactory(factory); } + + /** + * @param factoryClass an {@link IInjectorFactory} implementation, instantiated with the object + * factory currently in use. + */ + public void setInjectorFactoryClass(Class factoryClass) { + setInjectorFactory(m_objectFactory.newInstance(factoryClass)); + } } diff --git a/testng-core/src/test/java/test/TestHelper.java b/testng-core/src/test/java/test/TestHelper.java deleted file mode 100644 index 11fec7b1d..000000000 --- a/testng-core/src/test/java/test/TestHelper.java +++ /dev/null @@ -1,56 +0,0 @@ -package test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import org.testng.TestNG; -import org.testng.xml.XmlClass; -import org.testng.xml.XmlSuite; -import org.testng.xml.XmlTest; - -public class TestHelper { - /* - * TestNG issues a warning if the XML misses DOCTYPE, so here's a common header for - * xml suites generated in the tests. - */ - public static final String SUITE_XML_HEADER = - "\n" - + "\n"; - - public static XmlSuite createSuite(String cls, String suiteName) { - XmlSuite result = new XmlSuite(); - result.setName(suiteName); - - XmlTest test = new XmlTest(result); - test.setName("TmpTest"); - test.setXmlClasses(Collections.singletonList(new XmlClass(cls))); - - return result; - } - - public static TestNG createTestNG() throws IOException { - return createTestNG(createRandomDirectory()); - } - - public static TestNG createTestNG(XmlSuite suite) throws IOException { - return createTestNG(suite, createRandomDirectory()); - } - - public static TestNG createTestNG(XmlSuite suite, Path outputDir) { - TestNG result = createTestNG(outputDir); - result.setXmlSuites(Collections.singletonList(suite)); - return result; - } - - private static TestNG createTestNG(Path outputDir) { - TestNG result = new TestNG(); - result.setOutputDirectory(outputDir.toAbsolutePath().toString()); - - return result; - } - - public static Path createRandomDirectory() throws IOException { - return Files.createTempDirectory("testng-tmp-"); - } -} diff --git a/testng-core/src/test/java/test/commandline/CommandLineOverridesXml.java b/testng-core/src/test/java/test/commandline/CommandLineOverridesXml.java index e6ba273c4..a75b2fa74 100644 --- a/testng-core/src/test/java/test/commandline/CommandLineOverridesXml.java +++ b/testng-core/src/test/java/test/commandline/CommandLineOverridesXml.java @@ -2,25 +2,14 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Set; import org.testng.TestListenerAdapter; import org.testng.TestNG; import org.testng.annotations.Test; -import org.testng.testhelper.JarCreator; import org.testng.xml.XmlSuite; import test.SimpleBaseTest; -import test.TestHelper; -import test.commandline.issue341.LocalLogAggregator; -import test.commandline.issue341.TestSampleA; -import test.commandline.issue341.TestSampleB; public class CommandLineOverridesXml extends SimpleBaseTest { @@ -63,47 +52,4 @@ public void ensureThatParallelismAndThreadCountAreRallied() { assertThat(Issue987TestSample.maps).hasSize(2); assertThat(Issue987TestSample.maps.values()).contains("method2", "method1"); } - - @Test(description = "GITHUB-341") - public void ensureParallelismIsHonoredWhenOnlyClassesSpecifiedInJar() throws IOException { - Class[] classes = new Class[] {TestSampleA.class, TestSampleB.class}; - File jarfile = JarCreator.generateJar(classes); - String[] args = - new String[] { - "-parallel", - "classes", - "-testjar", - jarfile.getAbsolutePath(), - "-listener", - LocalLogAggregator.class.getCanonicalName() - }; - TestNG.privateMain(args, null); - Set logs = LocalLogAggregator.getLogs(); - assertThat(logs).hasSize(2); - } - - @Test(description = "GITHUB-1810") - public void ensureNoNullPointerExceptionIsThrown() throws IOException { - TestNG testng = TestNG.privateMain(new String[] {createTemporarySuiteAndGetItsPath()}, null); - assertThat(testng.getStatus()).isEqualTo(8); - } - - private static String createTemporarySuiteAndGetItsPath() throws IOException { - Path file = Files.createTempFile("testng", ".xml"); - Files.write( - file, buildSuiteContentThatRefersToInvalidTestClass().getBytes(StandardCharsets.UTF_8)); - return file.toFile().getAbsolutePath(); - } - - private static String buildSuiteContentThatRefersToInvalidTestClass() { - return TestHelper.SUITE_XML_HEADER - + "\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " \n" - + " \n" - + "\n"; - } } diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java b/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java index f3cf45ce7..09eebcd2e 100644 --- a/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java +++ b/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java @@ -17,6 +17,7 @@ import org.testng.testhelper.OutputDirectoryPatch; import org.testng.xml.XmlSuite; import test.SimpleBaseTest; +import test.TestHelper; import test.configurationfailurepolicy.issue2862.AnnotationAtParentClassLevelForMethodConfigSample; import test.configurationfailurepolicy.issue2862.AnnotationAtParentClassLevelForMethodConfigSample2; import test.configurationfailurepolicy.issue2862.AnnotationAtParentClassLevelForMethodConfigSample3; @@ -83,7 +84,7 @@ public void confFailureTest( testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); testng.run(); - verify(tla, configurationFailures, configurationSkips, skippedTests); + TestHelper.assertCounts(tla, configurationFailures, configurationSkips, skippedTests); } @Test @@ -99,97 +100,7 @@ public void confFailureTestInvolvingGroups() { testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); testng.setGroups("group1"); testng.run(); - verify(tla, 1, 3, 1); - } - - @Test - public void commandLineTest_policyAsSkip() { - String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "skip", - "-testclass", - ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() - }; - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG.privateMain(argv, tla); - - verify(tla, 1, 1, 2); - } - - @Test - public void commandLineTest_policyAsContinue() { - String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "continue", - "-testclass", - ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() - }; - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG.privateMain(argv, tla); - - verify(tla, 2, 0, 2); - } - - @Test - public void commandLineTestWithXMLFile_policyAsSkip() { - String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "skip", - getPathToResource("testng-configfailure.xml") - }; - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG.privateMain(argv, tla); - - verify(tla, 1, 1, 2); - } - - @Test - public void commandLineTestWithXMLFile_policyAsContinue() { - String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "continue", - getPathToResource("testng-configfailure.xml") - }; - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG.privateMain(argv, tla); - - verify(tla, 2, 0, 2); - } - - private void verify( - TestListenerAdapter tla, - int configurationFailures, - int configurationSkips, - int skippedTests) { - assertThat(tla.getConfigurationFailures().size()) - .withFailMessage("wrong number of configuration failures") - .isEqualTo(configurationFailures); - assertThat(tla.getConfigurationSkips().size()) - .withFailMessage("wrong number of configuration skips") - .isEqualTo(configurationSkips); - assertThat(tla.getSkippedTests().size()) - .withFailMessage("wrong number of skipped tests") - .isEqualTo(skippedTests); + TestHelper.assertCounts(tla, 1, 3, 1); } @Test(description = "GITHUB-2862") diff --git a/testng-core/src/test/java/test/github1417/YetAnotherTestClassSample.java b/testng-core/src/test/java/test/github1417/YetAnotherTestClassSample.java index 26f98b324..6a0acd720 100644 --- a/testng-core/src/test/java/test/github1417/YetAnotherTestClassSample.java +++ b/testng-core/src/test/java/test/github1417/YetAnotherTestClassSample.java @@ -2,12 +2,12 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.beust.jcommander.internal.Lists; import java.util.List; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import org.testng.collections.Lists; public class YetAnotherTestClassSample { private static YetAnotherTestClassSample instance; diff --git a/testng-core/src/test/java/test/groups/issue2232/IssueTest.java b/testng-core/src/test/java/test/groups/issue2232/IssueTest.java index e4c1f026b..382e08241 100644 --- a/testng-core/src/test/java/test/groups/issue2232/IssueTest.java +++ b/testng-core/src/test/java/test/groups/issue2232/IssueTest.java @@ -2,22 +2,8 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import org.testng.Reporter; import org.testng.TestNG; import org.testng.annotations.Test; -import org.testng.xml.XmlGroups; -import org.testng.xml.XmlPackage; -import org.testng.xml.XmlRun; -import org.testng.xml.XmlSuite; -import org.testng.xml.XmlTest; import test.SimpleBaseTest; public class IssueTest extends SimpleBaseTest { @@ -26,60 +12,8 @@ public class IssueTest extends SimpleBaseTest { // sporadic and is not easy to reproduce. That is why this test is being executed 10 times // to ensure that the issue can be reproduced in one of the executions public void ensureNoNPEThrownWhenRunningGroups() throws InterruptedException { - TestNG testng = create(constructSuite()); + TestNG testng = create(Issue2232Suites.construct()); testng.run(); assertThat(testng.getStatus()).isEqualTo(0); } - - private XmlSuite constructSuite() { - XmlSuite xmlsuite = createXmlSuite("2232_suite"); - xmlsuite.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); - xmlsuite.setThreadCount(256); - xmlsuite.setParallel(XmlSuite.ParallelMode.CLASSES); - XmlTest xmltest = createXmlTest(xmlsuite, "2232_test"); - XmlRun xmlrun = new XmlRun(); - xmlrun.onInclude("Group2"); - xmlrun.onExclude("Broken"); - XmlGroups xmlgroup = new XmlGroups(); - xmlgroup.setRun(xmlrun); - xmltest.setGroups(xmlgroup); - XmlPackage xmlpackage = new XmlPackage(); - xmlpackage.setName(getClass().getPackage().getName() + ".samples.*"); - xmltest.setPackages(Collections.singletonList(xmlpackage)); - return xmlsuite; - } - - @Test(invocationCount = 2, description = "GITHUB-2232") - // Ensuring that the bug doesn't surface even when tests are executed via the command line mode - public void commandlineTest() throws IOException, InterruptedException { - Path suitefile = - Files.write( - Files.createTempFile("testng", ".xml"), - constructSuite().toXml().getBytes(Charset.defaultCharset())); - List args = Collections.singletonList(suitefile.toFile().getAbsolutePath()); - int status = exec(Collections.emptyList(), args); - assertThat(status).isEqualTo(0); - } - - private int exec(List jvmArgs, List args) - throws IOException, InterruptedException { - - String javaHome = System.getProperty("java.home"); - String javaBin = javaHome + File.separator + "bin" + File.separator + "java"; - String classpath = System.getProperty("java.class.path"); - String className = TestNG.class.getName(); - List command = new ArrayList<>(); - command.add(javaBin); - command.addAll(jvmArgs); - command.add("-cp"); - command.add(classpath); - command.add(className); - command.addAll(args); - Reporter.log("Executing the command " + command, 2, true); - ProcessBuilder builder = new ProcessBuilder(command); - Process process = builder.inheritIO().start(); - process.waitFor(); - - return process.exitValue(); - } } diff --git a/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java b/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java index c4c5e05ab..767113014 100644 --- a/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java +++ b/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java @@ -5,7 +5,6 @@ import static org.assertj.core.api.Assertions.fail; import static test.junitreports.TestClassContainerForGithubIssue1265.*; -import com.beust.jcommander.internal.Lists; import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -25,6 +24,7 @@ import org.testng.Reporter; import org.testng.TestNG; import org.testng.annotations.Test; +import org.testng.collections.Lists; import org.testng.collections.Maps; import org.testng.reporters.XMLConstants; import org.testng.xml.XmlSuite; diff --git a/testng-core/src/test/java/test/listeners/ListenersTest.java b/testng-core/src/test/java/test/listeners/ListenersTest.java index 9afae8ced..5ce7828f6 100644 --- a/testng-core/src/test/java/test/listeners/ListenersTest.java +++ b/testng-core/src/test/java/test/listeners/ListenersTest.java @@ -335,96 +335,6 @@ public void ensureOrderingForMethodInterceptorsViaXmlTag() { MethodInterceptorHolder.EXPECTED_LOGS); } - @Test(description = "GITHUB-2916") - public void ensureOrderingForExecutionListenersViaCli() { - Ensure.orderingViaCli( - ExecutionListenerHolder.LOGS, - ExecutionListenerHolder.ALL_STRING, - ExecutionListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForAlterSuiteListenersViaCli() { - Ensure.orderingViaCli( - AlterSuiteListenerHolder.LOGS, - AlterSuiteListenerHolder.ALL_STRING, - AlterSuiteListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForSuiteListenersViaCli() { - Ensure.orderingViaCli( - SuiteListenerHolder.LOGS, - SuiteListenerHolder.ALL_STRING, - SuiteListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForTestListenersViaCli() { - Ensure.orderingViaCli( - ElaborateSampleTestCase.class, TestListenerHolder.LOGS, - TestListenerHolder.ALL_STRING, TestListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForInvokedMethodListenersViaCli() { - Ensure.orderingViaCli( - InvokedMethodListenerHolder.LOGS, - InvokedMethodListenerHolder.ALL_STRING, - InvokedMethodListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForConfigurationListenersViaCli() { - Ensure.orderingViaCli( - SimpleConfigTestCase.class, - ConfigurationListenerHolder.LOGS, - ConfigurationListenerHolder.ALL_STRING, - ConfigurationListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForClassListenersViaCli() { - Ensure.orderingViaCli( - ClassListenerHolder.LOGS, - ClassListenerHolder.ALL_STRING, - ClassListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForDataProviderListenersViaCli() { - Ensure.orderingViaCli( - DataProviderSampleTestCase.class, - DataProviderListenerHolder.LOGS, - DataProviderListenerHolder.ALL_STRING, - DataProviderListenerHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForDataProviderInterceptorsViaCli() { - Ensure.orderingViaCli( - DataProviderSampleTestCase.class, - DataProviderInterceptorHolder.LOGS, - DataProviderInterceptorHolder.ALL_STRING, - DataProviderInterceptorHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForExecutionVisualisersViaCli() { - Ensure.orderingViaCli( - ExecutionVisualiserHolder.LOGS, - ExecutionVisualiserHolder.ALL_STRING, - ExecutionVisualiserHolder.EXPECTED_LOGS); - } - - @Test(description = "GITHUB-2916") - public void ensureOrderingForMethodInterceptorsViaCli() { - Ensure.orderingViaCli( - MethodInterceptorHolder.LOGS, - MethodInterceptorHolder.ALL_STRING, - MethodInterceptorHolder.EXPECTED_LOGS); - } - @Test(description = "GITHUB-2916") public void ensureOrderingForExecutionListenersViaAnnotation() { // This is a special case scenario and so the expected value order is different from the @@ -635,23 +545,6 @@ public void ensureListenersCanBeDisabled() { assertThat(SampleTransformer.getLogs()).isEmpty(); } - @Test(description = "GITHUB-2381") - public void ensureListenersCanBeDisabledViaCLI() { - SampleGlobalListener.clearLogs(); - SampleTransformer.clearLogs(); - String listeners = - String.join(",", SampleGlobalListener.class.getName(), SampleTransformer.class.getName()); - String testClasses = - String.join( - ",", - test.listeners.issue2381.TestClassSample.class.getName(), - FactoryTestClassSample.class.getName()); - List args = List.of("-listener", listeners, "-testclass", testClasses); - TestNG.privateMain(args.toArray(String[]::new), null); - assertThat(SampleGlobalListener.getLogs()).isEmpty(); - assertThat(SampleTransformer.getLogs()).isEmpty(); - } - @Test(description = "GITHUB-2381") public void ensureListenersCanBeDisabledViaSuiteFile() { SampleGlobalListener.clearLogs(); @@ -866,24 +759,6 @@ static void orderingViaAnnotation( assertThat(logs).containsExactly(expected); } - static void orderingViaCli(List logs, List listeners, String[] expected) { - orderingViaCli(NormalSampleTestCase.class, logs, listeners, expected); - } - - static void orderingViaCli( - Class clazz, List logs, List listeners, String[] expected) { - logs.clear(); - String[] args = - new String[] { - "-listener", String.join(",", listeners), - "-usedefaultlisteners", "false", - "-testclass", clazz.getName(), - "-listenercomparator", AnnotationBackedListenerComparator.class.getName() - }; - TestNG.privateMain(args, null); - assertThat(logs).containsExactly(expected); - } - static void orderingViaXmlTag(List logs, List listeners, String[] expected) { orderingViaXmlTag(NormalSampleTestCase.class, logs, listeners, expected); } diff --git a/testng-core/src/test/java/test/listeners/factory/TestNGFactoryTest.java b/testng-core/src/test/java/test/listeners/factory/TestNGFactoryTest.java index e810364d0..412f25b9e 100644 --- a/testng-core/src/test/java/test/listeners/factory/TestNGFactoryTest.java +++ b/testng-core/src/test/java/test/listeners/factory/TestNGFactoryTest.java @@ -4,7 +4,6 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import org.testng.CommandLineArgs; import org.testng.ITestListener; import org.testng.ITestResult; import org.testng.TestNG; @@ -16,23 +15,6 @@ public class TestNGFactoryTest extends SimpleBaseTest { - @Test(description = "GITHUB-3059") - public void testListenerFactoryViaConfigurationArg() { - String[] args = - new String[] { - CommandLineArgs.LISTENER_FACTORY, - SampleTestFactory.class.getName(), - CommandLineArgs.TEST_CLASS, - SampleTestCase.class.getName(), - CommandLineArgs.LISTENER, - ExampleListener.class.getName() - }; - TestNG testng = TestNG.privateMain(args, null); - assertThat(SampleTestFactory.instance).isNotNull(); - assertThat(ExampleListener.getInstance()).isNotNull(); - assertThat(testng.getStatus()).isZero(); - } - @Test(description = "GITHUB-3059") public void testListenerFactoryViaTestNGApi() { TestNG testng = new TestNG(); diff --git a/testng-core/src/test/java/test/methodselectors/MethodSelectorInSuiteTest.java b/testng-core/src/test/java/test/methodselectors/MethodSelectorInSuiteTest.java index 7ac43a3f5..658cceb27 100644 --- a/testng-core/src/test/java/test/methodselectors/MethodSelectorInSuiteTest.java +++ b/testng-core/src/test/java/test/methodselectors/MethodSelectorInSuiteTest.java @@ -1,22 +1,19 @@ package test.methodselectors; -import static org.assertj.core.api.Assertions.assertThat; - import java.util.Collections; import java.util.List; import org.testng.ITestNGListener; -import org.testng.ITestResult; import org.testng.TestListenerAdapter; import org.testng.TestNG; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.testng.collections.Lists; -import org.testng.testhelper.OutputDirectoryPatch; import org.testng.xml.XmlClass; import org.testng.xml.XmlMethodSelector; import org.testng.xml.XmlSuite; import org.testng.xml.XmlTest; import test.SimpleBaseTest; +import test.TestHelper; public class MethodSelectorInSuiteTest extends SimpleBaseTest { @@ -44,7 +41,7 @@ public void programmaticXmlSuite() { tng.addListener((ITestNGListener) m_tla); tng.run(); - validate(new String[] {"test2"}); + TestHelper.assertPassedTestNames(m_tla.getPassedTests(), "test2"); } @Test @@ -54,28 +51,6 @@ public void xmlXmlSuite() { tng.addListener((ITestNGListener) m_tla); tng.run(); - validate(new String[] {"test2"}); - } - - @Test - public void fileOnCommandLine() { - String[] args = - new String[] { - "-d", - OutputDirectoryPatch.getOutputDirectory(), - getPathToResource("methodselector-in-xml.xml") - }; - TestNG.privateMain(args, m_tla); - - validate(new String[] {"test2"}); - } - - private void validate(String[] expectPassed) { - List passed = m_tla.getPassedTests(); - assertThat(passed.size()).isEqualTo(expectPassed.length); - // doing this index based is probably not the best - for (int i = 0; i < expectPassed.length; i++) { - assertThat(passed.get(i).getName()).isEqualTo(expectPassed[i]); - } + TestHelper.assertPassedTestNames(m_tla.getPassedTests(), "test2"); } } diff --git a/testng-core/src/test/java/test/reports/EmailableReporterTest.java b/testng-core/src/test/java/test/reports/EmailableReporterTest.java index 2bcce4631..0821ab7c3 100644 --- a/testng-core/src/test/java/test/reports/EmailableReporterTest.java +++ b/testng-core/src/test/java/test/reports/EmailableReporterTest.java @@ -28,17 +28,6 @@ public void testReportsNameCustomizationViaRunMethodInvocation(IReporter reporte runTestViaRunMethod(reporter, null /* no jvm arguments */); } - @Test(dataProvider = "getReporterNames", priority = 3) - public void testReportsNameCustomizationViaMainMethodInvocation(String clazzName) { - runTestViaMainMethod(clazzName, null /* no jvm arguments */); - } - - @Test(dataProvider = "getReporterNames", priority = 4) - public void testReportsNameCustomizationViaMainMethodInvocationAndJVMArguments( - String clazzName, String jvm) { - runTestViaMainMethod(clazzName, jvm); - } - @Test public void ensureEmailableReportsDontThrowExceptions() { runTest(TestCaseSample.class); @@ -71,42 +60,6 @@ public Object[][] getReporterInstances(Method method) { return new Object[][] {{new EmailableReporter2()}}; } - @DataProvider(name = "getReporterNames") - public Object[][] getReporterNames(Method method) { - if (method.getName().toLowerCase().contains("jvmarguments")) { - return new Object[][] {{EmailableReporter2.class.getName(), "emailable.report2.name"}}; - } - return new Object[][] {{EmailableReporter2.class.getName()}}; - } - - private void runTestViaMainMethod(String clazzName, String jvm) { - String name = Long.toString(System.currentTimeMillis()); - File output = createDirInTempDir(name); - String filename = "report" + name + ".html"; - String[] args = { - "-d", - output.getAbsolutePath(), - "-reporter", - clazzName + ":fileName=" + filename, - "src/test/resources/1332.xml" - }; - try { - if (jvm != null) { - System.setProperty(jvm, filename); - } - TestNG.privateMain(args, null); - } catch (SecurityException t) { - // Gobble Security exception - } finally { - if (jvm != null) { - // reset the jvm arguments - System.setProperty(jvm, ""); - } - } - File actual = new File(output.getAbsolutePath(), filename); - assertThat(actual).exists(); - } - private void runTestViaRunMethod(IReporter reporter, String jvm) { String name = Long.toString(System.currentTimeMillis()); File output = createDirInTempDir(name); diff --git a/testng-core/src/test/java/test/testng1231/TestExecutionListenerInvocationOrder.java b/testng-core/src/test/java/test/testng1231/TestExecutionListenerInvocationOrder.java index eab74633a..3792d1d43 100644 --- a/testng-core/src/test/java/test/testng1231/TestExecutionListenerInvocationOrder.java +++ b/testng-core/src/test/java/test/testng1231/TestExecutionListenerInvocationOrder.java @@ -2,12 +2,11 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.beust.jcommander.internal.Lists; import java.util.Arrays; -import java.util.LinkedList; import java.util.List; import org.testng.*; import org.testng.annotations.Test; +import org.testng.collections.Lists; import org.testng.xml.XmlSuite; import org.testng.xml.XmlTest; import test.SimpleBaseTest; @@ -28,7 +27,7 @@ public void testListenerOrder() { public static class TestListenerFor1231 implements IExecutionListener, IAlterSuiteListener, IReporter, ISuiteListener { - public static LinkedList order = Lists.newLinkedList(); + public static List order = Lists.newLinkedList(); @Override public void onExecutionStart() { diff --git a/testng-core/src/test/java/test/thread/CustomExecutorServiceFactoryTest.java b/testng-core/src/test/java/test/thread/CustomExecutorServiceFactoryTest.java index b371166b8..255b48352 100644 --- a/testng-core/src/test/java/test/thread/CustomExecutorServiceFactoryTest.java +++ b/testng-core/src/test/java/test/thread/CustomExecutorServiceFactoryTest.java @@ -2,20 +2,10 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; import org.testng.TestNG; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -import org.testng.collections.Lists; import org.testng.xml.XmlSuite; import test.SimpleBaseTest; import test.thread.issue3066.Issue3066ExecutorServiceFactory; @@ -24,20 +14,6 @@ public class CustomExecutorServiceFactoryTest extends SimpleBaseTest { - @Test(description = "GITHUB-3066") - public void ensureCanWireInCustomExecutorServiceWhenEnabledViaConfigParam() { - String[] args = { - "-testclass", - TestClassSample.class.getName(), - "-threadpoolfactoryclass", - Issue3066ExecutorServiceFactory.class.getName(), - "-parallel", - "methods" - }; - TestNG.privateMain(args, null); - assertThat(Issue3066ThreadPoolExecutor.isInvoked()).isTrue(); - } - @Test(description = "GITHUB-3066") public void ensureCanWireInCustomExecutorServiceWhenEnabledViaAPI() { TestNG testng = create(TestClassSample.class); @@ -59,35 +35,6 @@ public void ensureCanWireInCustomExecutorServiceWhenEnabledViaAPIForMultipleSuit assertThat(Issue3066ThreadPoolExecutor.isInvoked()).isTrue(); } - @Test(description = "GITHUB-3066") - public void ensureCanWireInCustomExecutorServiceWhenEnabledViaConfigForMultipleSuites() { - AtomicInteger counter = new AtomicInteger(1); - List suites = new ArrayList<>(); - File dir = createDirInTempDir("suites"); - Stream.of(TestClassSample.class, TestClassSample.class) - .map( - it -> createXmlSuite("suite-" + counter.get(), "test-" + counter.getAndIncrement(), it)) - .map(XmlSuite::toXml) - .forEach( - it -> { - Path s1 = Paths.get(dir.getAbsolutePath(), UUID.randomUUID() + "-suite.xml"); - try { - Files.writeString(s1, it); - suites.add(s1.toFile().getAbsolutePath()); - } catch (IOException ignored) { - } - }); - - List args = - List.of( - "-threadpoolfactoryclass", - Issue3066ExecutorServiceFactory.class.getName(), - "-suitethreadpoolsize", - "2"); - TestNG.privateMain(Lists.merge(suites, args).toArray(String[]::new), null); - assertThat(Issue3066ThreadPoolExecutor.isInvoked()).isTrue(); - } - @AfterMethod public void resetState() { Issue3066ThreadPoolExecutor.resetInvokedState(); diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml index 5ad70b2f7..6ad4b7b47 100644 --- a/testng-core/src/test/resources/testng.xml +++ b/testng-core/src/test/resources/testng.xml @@ -586,7 +586,6 @@ - diff --git a/testng-core/testng-core-build.gradle.kts b/testng-core/testng-core-build.gradle.kts index 109c434a9..ef891d446 100644 --- a/testng-core/testng-core-build.gradle.kts +++ b/testng-core/testng-core-build.gradle.kts @@ -24,7 +24,6 @@ dependencies { api(projects.testngCoreApi) // Annotations have to be available on the compile classpath for the proper compilation compileOnly("com.github.spotbugs:spotbugs:4.10.3") - api("org.jcommander:jcommander:2.0") "guiceApi"(platform("com.google.inject:guice-bom:6.0.0")) "guiceApi"("com.google.inject:guice") @@ -34,13 +33,12 @@ dependencies { implementation(projects.testngReflectionUtils) implementation(projects.testngRunnerApi) testImplementation("org.testng:testng-asserts:1.0.0") + testImplementation(projects.testngTestKit) testImplementation("org.apache.groovy:groovy-all:5.0.7") { exclude("org.testng", "testng") } testImplementation("org.apache-extras.beanshell:bsh:2.0b6") testImplementation("org.mockito:mockito-core:5.23.0") - testImplementation("org.jboss.shrinkwrap:shrinkwrap-api:1.2.6") - testImplementation("org.jboss.shrinkwrap:shrinkwrap-impl-base:1.2.6") testImplementation("org.xmlunit:xmlunit-assertj:2.12.0") testImplementation("in.jlibs:jlibs-core:3.0.1") testImplementation("org.gridkit.jvmtool:heaplib:0.2") diff --git a/testng-core/src/main/java/org/testng/Converter.java b/testng-jcommander/src/main/java/org/testng/cli/jcommander/Converter.java similarity index 93% rename from testng-core/src/main/java/org/testng/Converter.java rename to testng-jcommander/src/main/java/org/testng/cli/jcommander/Converter.java index 2a150e1d5..a77156d67 100644 --- a/testng-core/src/main/java/org/testng/Converter.java +++ b/testng-jcommander/src/main/java/org/testng/cli/jcommander/Converter.java @@ -1,4 +1,4 @@ -package org.testng; +package org.testng.cli.jcommander; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -10,6 +10,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import org.testng.TestNGException; import org.testng.collections.Sets; import org.testng.internal.Yaml; import org.testng.xml.XmlSuite; @@ -18,6 +19,9 @@ /** * Convert XML files to YAML and vice versa. * + *

Moved out of the {@code org.testng} package in 7.13, when the command line front end was + * extracted from {@code testng-core}. + * * @author cbeust */ public class Converter { diff --git a/testng-jcommander/src/main/java/org/testng/cli/jcommander/JCommanderCliRunner.java b/testng-jcommander/src/main/java/org/testng/cli/jcommander/JCommanderCliRunner.java new file mode 100644 index 000000000..a19e9f964 --- /dev/null +++ b/testng-jcommander/src/main/java/org/testng/cli/jcommander/JCommanderCliRunner.java @@ -0,0 +1,32 @@ +package org.testng.cli.jcommander; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import org.testng.cli.AbstractCliRunner; +import org.testng.cli.CliOptions; +import org.testng.cli.CliParseException; + +/** + * The JCommander backed implementation of {@link org.testng.ITestNGCliRunner}, wired in through + * {@code META-INF/services}. + * + * @since 7.13 + */ +public class JCommanderCliRunner extends AbstractCliRunner { + + @Override + protected CliOptions parse(String[] argv) { + JCommanderOptions options = new JCommanderOptions(); + try { + new JCommander(options).parse(argv); + } catch (ParameterException ex) { + throw new CliParseException(ex.getMessage(), ex); + } + return options.toCliOptions(); + } + + @Override + public void usage() { + new JCommander(new JCommanderOptions()).usage(); + } +} diff --git a/testng-jcommander/src/main/java/org/testng/cli/jcommander/JCommanderOptions.java b/testng-jcommander/src/main/java/org/testng/cli/jcommander/JCommanderOptions.java new file mode 100644 index 000000000..916e84144 --- /dev/null +++ b/testng-jcommander/src/main/java/org/testng/cli/jcommander/JCommanderOptions.java @@ -0,0 +1,270 @@ +package org.testng.cli.jcommander; + +import com.beust.jcommander.Parameter; +import java.util.List; +import org.testng.cli.CliOptions; +import org.testng.collections.Lists; +import org.testng.xml.XmlSuite; + +/** + * The JCommander description of the TestNG command line. Its only job is to be filled by JCommander + * and handed over as a {@link CliOptions}. + * + * @since 7.13 + */ +public class JCommanderOptions { + + @Parameter(description = "The XML suite files to run") + public List suiteFiles = Lists.newArrayList(); + + @Parameter( + names = {CliOptions.LOG, CliOptions.VERBOSE}, + description = "Level of verbosity") + public Integer verbose; + + @Parameter( + names = CliOptions.GROUPS, + description = "Comma-separated list of group names to be run") + public String groups; + + @Parameter( + names = CliOptions.EXCLUDED_GROUPS, + description = "Comma-separated list of group names to " + " exclude") + public String excludedGroups; + + @Parameter(names = CliOptions.OUTPUT_DIRECTORY, description = "Output directory") + public String outputDirectory; + + /** Accepted for backwards compatibility; TestNG has never acted on it. */ + @Parameter( + names = CliOptions.MIXED, + description = + "No-op since JUnit execution support was removed in 7.10.0." + + " Kept for command line backward compatibility.") + public Boolean mixed = Boolean.FALSE; + + @Parameter( + names = CliOptions.LISTENER, + description = + "List of .class files or list of class names" + + " implementing ITestListener or ISuiteListener") + public String listener; + + @Parameter( + names = CliOptions.LISTENER_COMPARATOR, + description = + "An implementation of ListenerComparator that will be used by TestNG to determine order of execution for listeners") + public String listenerComparator; + + @Parameter( + names = CliOptions.METHOD_SELECTORS, + description = "List of .class files or list of class " + "names implementing IMethodSelector") + public String methodSelectors; + + @Parameter( + names = CliOptions.OBJECT_FACTORY, + description = + "Fully qualified class name that implements org.testng.ITestObjectFactory which can be used to create test class and listener instances.") + public String objectFactory; + + @Parameter(names = CliOptions.PARALLEL, description = "Parallel mode (methods, tests or classes)") + public XmlSuite.ParallelMode parallelMode; + + @Parameter( + names = CliOptions.CONFIG_FAILURE_POLICY, + description = "Configuration failure policy (skip or continue)") + public String configFailurePolicy; + + @Parameter( + names = CliOptions.THREAD_COUNT, + description = "Number of threads to use when running tests " + "in parallel") + public Integer threadCount; + + @Parameter( + names = CliOptions.DATA_PROVIDER_THREAD_COUNT, + description = "Number of threads to use when " + "running data providers") + public Integer dataProviderThreadCount; + + @Parameter( + names = CliOptions.SUITE_NAME, + description = + "Default name of test suite, if not specified " + + "in suite definition file or source code") + public String suiteName; + + @Parameter( + names = CliOptions.TEST_NAME, + description = + "Default name of test, if not specified in suite" + "definition file or source code") + public String testName; + + @Parameter( + names = CliOptions.REPORTER, + description = "Extended configuration for custom report listener") + public String reporter; + + @Parameter( + names = CliOptions.USE_DEFAULT_LISTENERS, + description = "Whether to use the default listeners") + public String useDefaultListeners = "true"; + + @Parameter(names = CliOptions.SKIP_FAILED_INVOCATION_COUNTS, hidden = true) + public Boolean skipFailedInvocationCounts; + + @Parameter(names = CliOptions.TEST_CLASS, description = "The list of test classes") + public String testClass; + + @Parameter(names = CliOptions.TEST_NAMES, description = "The list of test names to run") + public String testNames; + + @Parameter( + names = CliOptions.IGNORE_MISSED_TEST_NAMES, + description = + "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.") + public boolean ignoreMissedTestNames = false; + + @Parameter(names = CliOptions.TEST_JAR, description = "A jar file containing the tests") + public String testJar; + + @Parameter( + names = CliOptions.XML_PATH_IN_JAR, + description = + "The full path to the xml file inside the jar file (only valid if -testjar was specified)") + public String xmlPathInJar = CliOptions.XML_PATH_IN_JAR_DEFAULT; + + @Parameter( + names = {CliOptions.TEST_RUNNER_FACTORY, "-testRunFactory"}, + description = "The factory used to create tests") + public String testRunnerFactory; + + @Parameter( + names = CliOptions.LISTENER_FACTORY, + description = "The factory used to create TestNG listeners") + public String listenerFactory; + + @Parameter(names = CliOptions.METHODS, description = "Comma separated of test methods") + public List commandLineMethods = Lists.newArrayList(); + + @Parameter( + names = CliOptions.SUITE_THREAD_POOL_SIZE, + description = "Size of the thread pool to use to run suites") + public Integer suiteThreadPoolSize = CliOptions.SUITE_THREAD_POOL_SIZE_DEFAULT; + + @Parameter( + names = CliOptions.RANDOMIZE_SUITES, + hidden = true, + description = "Whether to run suites in same order as specified in XML or not") + public Boolean randomizeSuites = Boolean.FALSE; + + @Parameter( + names = CliOptions.ALWAYS_RUN_LISTENERS, + description = "Should MethodInvocation Listeners be run even for skipped methods") + public Boolean alwaysRunListeners = Boolean.TRUE; + + @Parameter( + names = CliOptions.THREAD_POOL_FACTORY_CLASS, + description = "The threadpool executor factory implementation that TestNG should use.") + public String threadPoolFactoryClass; + + @Parameter( + names = CliOptions.DEPENDENCY_INJECTOR_FACTORY, + description = "The dependency injector factory implementation that TestNG should use.") + public String dependencyInjectorFactoryClass; + + @Parameter( + names = CliOptions.FAIL_IF_ALL_TESTS_SKIPPED, + description = "Should TestNG fail execution if all tests were skipped and nothing was run.") + public Boolean failIfAllTestsSkipped = false; + + @Parameter( + names = CliOptions.LISTENERS_TO_SKIP_VIA_SPI, + description = + "Comma separated fully qualified class names of listeners that should be skipped from being wired in via Service Loaders.") + public String spiListenersToSkip = ""; + + @Parameter( + names = CliOptions.OVERRIDE_INCLUDED_METHODS, + description = + "Whether command line method inclusions override the ones declared in the suite XML.") + public Boolean overrideIncludedMethods = false; + + @Parameter( + names = CliOptions.INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING, + description = + "Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.") + public Boolean includeAllDataDrivenTestsWhenSkipping = false; + + @Parameter( + names = CliOptions.PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE, + description = "Should TestNG consider failures in Data Providers as test failures.") + public Boolean propagateDataProviderFailureAsTestFailure = false; + + @Parameter( + names = CliOptions.GENERATE_RESULTS_PER_SUITE, + description = + "Should TestNG generate results on a per suite basis by creating a sub directory for each suite and dumping results into it.") + public Boolean generateResultsPerSuite = false; + + @Parameter( + names = CliOptions.SHARE_THREAD_POOL_FOR_DATA_PROVIDERS, + description = + "Should TestNG use a global Shared ThreadPool (At suite level) for running data providers.") + public Boolean shareThreadPoolForDataProviders = false; + + @Parameter( + names = CliOptions.USE_GLOBAL_THREAD_POOL, + description = + "Should TestNG use a global Shared ThreadPool (At suite level) for running regular and data driven tests.") + public Boolean useGlobalThreadPool = false; + + /** + * Converts this command line into the parser agnostic representation. + * + * @return the values TestNG configures itself from. + */ + public CliOptions toCliOptions() { + CliOptions cli = new CliOptions(); + // Copied rather than aliased: the returned options outlive this parser, and TestNG stores the + // lists by reference, so sharing them would let a later mutation change a running suite. + cli.suiteFiles = Lists.newArrayList(suiteFiles); + cli.verbose = verbose; + cli.groups = groups; + cli.excludedGroups = excludedGroups; + cli.outputDirectory = outputDirectory; + cli.listener = listener; + cli.listenerComparator = listenerComparator; + cli.methodSelectors = methodSelectors; + cli.objectFactory = objectFactory; + cli.parallelMode = parallelMode; + cli.configFailurePolicy = configFailurePolicy; + cli.threadCount = threadCount; + cli.dataProviderThreadCount = dataProviderThreadCount; + cli.suiteName = suiteName; + cli.testName = testName; + cli.reporter = reporter; + cli.useDefaultListeners = useDefaultListeners; + cli.skipFailedInvocationCounts = skipFailedInvocationCounts; + cli.testClass = testClass; + cli.testNames = testNames; + cli.ignoreMissedTestNames = ignoreMissedTestNames; + cli.testJar = testJar; + cli.xmlPathInJar = xmlPathInJar; + cli.testRunnerFactory = testRunnerFactory; + cli.listenerFactory = listenerFactory; + cli.commandLineMethods = Lists.newArrayList(commandLineMethods); + cli.suiteThreadPoolSize = suiteThreadPoolSize; + cli.randomizeSuites = randomizeSuites; + cli.alwaysRunListeners = alwaysRunListeners; + cli.threadPoolFactoryClass = threadPoolFactoryClass; + cli.dependencyInjectorFactoryClass = dependencyInjectorFactoryClass; + cli.failIfAllTestsSkipped = failIfAllTestsSkipped; + cli.spiListenersToSkip = spiListenersToSkip; + cli.overrideIncludedMethods = overrideIncludedMethods; + cli.includeAllDataDrivenTestsWhenSkipping = includeAllDataDrivenTestsWhenSkipping; + cli.propagateDataProviderFailureAsTestFailure = propagateDataProviderFailureAsTestFailure; + cli.generateResultsPerSuite = generateResultsPerSuite; + cli.shareThreadPoolForDataProviders = shareThreadPoolForDataProviders; + cli.useGlobalThreadPool = useGlobalThreadPool; + return cli; + } +} diff --git a/testng-jcommander/src/main/resources/META-INF/services/org.testng.ITestNGCliRunner b/testng-jcommander/src/main/resources/META-INF/services/org.testng.ITestNGCliRunner new file mode 100644 index 000000000..4dc600e4a --- /dev/null +++ b/testng-jcommander/src/main/resources/META-INF/services/org.testng.ITestNGCliRunner @@ -0,0 +1 @@ +org.testng.cli.jcommander.JCommanderCliRunner diff --git a/testng-jcommander/src/test/java/org/testng/cli/jcommander/JCommanderCliRunnerTest.java b/testng-jcommander/src/test/java/org/testng/cli/jcommander/JCommanderCliRunnerTest.java new file mode 100644 index 000000000..4496bfc9a --- /dev/null +++ b/testng-jcommander/src/test/java/org/testng/cli/jcommander/JCommanderCliRunnerTest.java @@ -0,0 +1,206 @@ +package org.testng.cli.jcommander; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.ServiceLoader; +import org.testng.ITestNGCliRunner; +import org.testng.annotations.Test; +import org.testng.cli.CliOptions; +import org.testng.cli.CliParseException; +import org.testng.xml.XmlSuite; + +public class JCommanderCliRunnerTest { + + private static CliOptions parse(String... argv) { + return new JCommanderCliRunner().parse(argv); + } + + @Test + public void serviceIsDiscoverable() { + ServiceLoader loader = + ServiceLoader.load(ITestNGCliRunner.class, ITestNGCliRunner.class.getClassLoader()); + assertThat(loader).hasAtLeastOneElementOfType(JCommanderCliRunner.class); + } + + @Test + public void suiteFilesArePositional() { + assertThat(parse("a.xml", "b.xml").suiteFiles).containsExactly("a.xml", "b.xml"); + } + + @Test + public void verboseAcceptsBothAliases() { + assertThat(parse("-log", "3", "a.xml").verbose).isEqualTo(3); + assertThat(parse("-verbose", "4", "a.xml").verbose).isEqualTo(4); + } + + @Test + public void testRunnerFactoryAcceptsBothSpellings() { + assertThat(parse("-testrunfactory", "com.acme.F", "a.xml").testRunnerFactory) + .isEqualTo("com.acme.F"); + assertThat(parse("-testRunFactory", "com.acme.F", "a.xml").testRunnerFactory) + .isEqualTo("com.acme.F"); + } + + @Test + public void parallelModeIsConvertedToAnEnum() { + assertThat(parse("-parallel", "methods", "a.xml").parallelMode) + .isEqualTo(XmlSuite.ParallelMode.METHODS); + } + + /** + * An unknown value must fail loudly. {@code XmlSuite.ParallelMode.getValidParallel} would + * silently degrade it to {@code NONE}, which would override what the suite XML declares. + */ + @Test + public void unknownParallelModeIsRejected() { + assertThatThrownBy(() -> parse("-parallel", "bogus", "a.xml")) + .isInstanceOf(CliParseException.class); + } + + /** + * Long standing behaviour: because the suite files are the main parameter, an unknown option is + * swallowed as a suite file name rather than rejected. Pinned here so that swapping the parser + * would surface the change. + */ + @Test + public void unknownOptionIsTreatedAsASuiteFile() { + assertThat(parse("-nosuchoption", "a.xml").suiteFiles) + .containsExactly("-nosuchoption", "a.xml"); + } + + @Test + public void missingValueForAKnownOptionIsRejected() { + assertThatThrownBy(() -> parse("-threadcount")).isInstanceOf(CliParseException.class); + } + + @Test + public void nonNumericValueForAKnownOptionIsRejected() { + assertThatThrownBy(() -> parse("-threadcount", "many", "a.xml")) + .isInstanceOf(CliParseException.class); + } + + /** {@code -usedefaultlisteners} keeps an arity of one, hence a String rather than a Boolean. */ + @Test + public void useDefaultListenersHasArityOne() { + assertThat(parse("a.xml").useDefaultListeners).isEqualTo("true"); + assertThat(parse("-usedefaultlisteners", "false", "a.xml").useDefaultListeners) + .isEqualTo("false"); + } + + @Test + public void defaultsAreCarriedOver() { + CliOptions cli = parse("a.xml"); + + assertThat(cli.spiListenersToSkip).isEmpty(); + assertThat(cli.xmlPathInJar).isEqualTo(CliOptions.XML_PATH_IN_JAR_DEFAULT); + assertThat(cli.suiteThreadPoolSize).isEqualTo(CliOptions.SUITE_THREAD_POOL_SIZE_DEFAULT); + assertThat(cli.alwaysRunListeners).isTrue(); + assertThat(cli.randomizeSuites).isFalse(); + assertThat(cli.commandLineMethods).isEmpty(); + } + + @Test + public void everyValueMakesItThroughTheMapping() { + CliOptions cli = + parse( + "-d", + "target/out", + "-groups", + "fast", + "-excludegroups", + "slow", + "-listener", + "com.acme.L1;com.acme.L2", + "-listenercomparator", + "com.acme.C", + "-methodselectors", + "com.acme.S:4", + "-objectfactory", + "com.acme.OF", + "-configfailurepolicy", + "continue", + "-threadcount", + "7", + "-dataproviderthreadcount", + "9", + "-suitename", + "aSuite", + "-testname", + "aTest", + "-reporter", + "com.acme.R:fileName=out.html", + "-testclass", + "com.acme.A,com.acme.B", + "-testnames", + "t1,t2", + "-ignoreMissedTestNames", + "-testjar", + "tests.jar", + "-xmlpathinjar", + "suites/all.xml", + "-listenerfactory", + "com.acme.LF", + "-methods", + "com.acme.A.m1", + "-suitethreadpoolsize", + "3", + "-threadpoolfactoryclass", + "com.acme.TP", + "-dependencyinjectorfactory", + "com.acme.DI", + "-spilistenerstoskip", + "com.acme.Skipped", + // The boolean flags matter most here: their CliOptions defaults equal their + // JCommanderOptions defaults, so a forgotten line in toCliOptions() stays invisible + // unless they are asserted in a non-default state. + "-skipfailedinvocationcounts", + "-randomizesuites", + "-failwheneverythingskipped", + "-overrideincludedmethods", + "-includeAllDataDrivenTestsWhenSkipping", + "-propagateDataProviderFailureAsTestFailure", + "-generateResultsPerSuite", + "-shareThreadPoolForDataProviders", + "-useGlobalThreadPool", + "a.xml"); + + assertThat(cli.outputDirectory).isEqualTo("target/out"); + assertThat(cli.groups).isEqualTo("fast"); + assertThat(cli.excludedGroups).isEqualTo("slow"); + assertThat(cli.listener).isEqualTo("com.acme.L1;com.acme.L2"); + assertThat(cli.listenerComparator).isEqualTo("com.acme.C"); + assertThat(cli.methodSelectors).isEqualTo("com.acme.S:4"); + assertThat(cli.objectFactory).isEqualTo("com.acme.OF"); + assertThat(cli.configFailurePolicy).isEqualTo("continue"); + assertThat(cli.threadCount).isEqualTo(7); + assertThat(cli.dataProviderThreadCount).isEqualTo(9); + assertThat(cli.suiteName).isEqualTo("aSuite"); + assertThat(cli.testName).isEqualTo("aTest"); + assertThat(cli.reporter).isEqualTo("com.acme.R:fileName=out.html"); + assertThat(cli.testClass).isEqualTo("com.acme.A,com.acme.B"); + assertThat(cli.testNames).isEqualTo("t1,t2"); + assertThat(cli.ignoreMissedTestNames).isTrue(); + assertThat(cli.testJar).isEqualTo("tests.jar"); + assertThat(cli.xmlPathInJar).isEqualTo("suites/all.xml"); + assertThat(cli.listenerFactory).isEqualTo("com.acme.LF"); + assertThat(cli.commandLineMethods).containsExactly("com.acme.A.m1"); + assertThat(cli.suiteThreadPoolSize).isEqualTo(3); + assertThat(cli.threadPoolFactoryClass).isEqualTo("com.acme.TP"); + assertThat(cli.dependencyInjectorFactoryClass).isEqualTo("com.acme.DI"); + assertThat(cli.spiListenersToSkip).isEqualTo("com.acme.Skipped"); + assertThat(cli.suiteFiles).containsExactly("a.xml"); + assertThat(cli.skipFailedInvocationCounts).isTrue(); + assertThat(cli.randomizeSuites).isTrue(); + // -alwaysrunlisteners is an arity-0 flag whose default is already TRUE, so the command line + // can never observe it in a non-default state, nor switch it off. Left as is for parity. + assertThat(cli.alwaysRunListeners).isTrue(); + assertThat(cli.failIfAllTestsSkipped).isTrue(); + assertThat(cli.overrideIncludedMethods).isTrue(); + assertThat(cli.includeAllDataDrivenTestsWhenSkipping).isTrue(); + assertThat(cli.propagateDataProviderFailureAsTestFailure).isTrue(); + assertThat(cli.generateResultsPerSuite).isTrue(); + assertThat(cli.shareThreadPoolForDataProviders).isTrue(); + assertThat(cli.useGlobalThreadPool).isTrue(); + } +} diff --git a/testng-jcommander/src/test/java/test/commandline/CommandLineOverridesXmlCommandLineTest.java b/testng-jcommander/src/test/java/test/commandline/CommandLineOverridesXmlCommandLineTest.java new file mode 100644 index 000000000..bbd8919f6 --- /dev/null +++ b/testng-jcommander/src/test/java/test/commandline/CommandLineOverridesXmlCommandLineTest.java @@ -0,0 +1,64 @@ +package test.commandline; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.Set; +import org.testng.TestNG; +import org.testng.annotations.Test; +import org.testng.testhelper.JarCreator; +import test.SimpleBaseTest; +import test.TestHelper; +import test.commandline.issue341.LocalLogAggregator; +import test.commandline.issue341.TestSampleA; +import test.commandline.issue341.TestSampleB; + +/** + * The command line half of {@code test.commandline.CommandLineOverridesXml}, which stays in {@code + * testng-core} for the cases that drive the Java API. + */ +public class CommandLineOverridesXmlCommandLineTest extends SimpleBaseTest { + + @Test(description = "GITHUB-341") + public void ensureParallelismIsHonoredWhenOnlyClassesSpecifiedInJar() throws IOException { + LocalLogAggregator.clearLogs(); + Class[] classes = new Class[] {TestSampleA.class, TestSampleB.class}; + File jarfile = JarCreator.generateJar(classes); + String[] args = + new String[] { + "-parallel", + "classes", + "-testjar", + jarfile.getAbsolutePath(), + "-listener", + LocalLogAggregator.class.getCanonicalName() + }; + TestNG.privateMain(args, null); + Set logs = LocalLogAggregator.getLogs(); + assertThat(logs).hasSize(2); + } + + @Test(description = "GITHUB-1810") + public void ensureNoNullPointerExceptionIsThrown() throws IOException { + TestNG testng = + TestNG.privateMain( + new String[] { + TestHelper.writeSuiteToTempFile(buildSuiteContentThatRefersToInvalidTestClass()) + }, + null); + assertThat(testng.getStatus()).isEqualTo(8); + } + + private static String buildSuiteContentThatRefersToInvalidTestClass() { + return TestHelper.SUITE_XML_HEADER + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n"; + } +} diff --git a/testng-jcommander/src/test/java/test/configurationfailurepolicy/FailurePolicyCommandLineTest.java b/testng-jcommander/src/test/java/test/configurationfailurepolicy/FailurePolicyCommandLineTest.java new file mode 100644 index 000000000..4c9066b2f --- /dev/null +++ b/testng-jcommander/src/test/java/test/configurationfailurepolicy/FailurePolicyCommandLineTest.java @@ -0,0 +1,90 @@ +package test.configurationfailurepolicy; + +import org.testng.TestListenerAdapter; +import org.testng.TestNG; +import org.testng.annotations.Test; +import org.testng.testhelper.OutputDirectoryPatch; +import test.SimpleBaseTest; +import test.TestHelper; + +/** + * The command line half of {@code test.configurationfailurepolicy.FailurePolicyTest}, which stays + * in {@code testng-core} for the cases that drive the Java API. These are the only tests covering + * the {@code -configfailurepolicy} string to {@code XmlSuite.FailurePolicy} conversion. + */ +public class FailurePolicyCommandLineTest extends SimpleBaseTest { + + @Test + public void commandLineTest_policyAsSkip() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "skip", + "-testclass", + ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + TestHelper.assertCounts(tla, 1, 1, 2); + } + + @Test + public void commandLineTest_policyAsContinue() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "continue", + "-testclass", + ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + TestHelper.assertCounts(tla, 2, 0, 2); + } + + @Test + public void commandLineTestWithXMLFile_policyAsSkip() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "skip", + getPathToResource("testng-configfailure.xml") + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + TestHelper.assertCounts(tla, 1, 1, 2); + } + + @Test + public void commandLineTestWithXMLFile_policyAsContinue() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "continue", + getPathToResource("testng-configfailure.xml") + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + TestHelper.assertCounts(tla, 2, 0, 2); + } +} diff --git a/testng-jcommander/src/test/java/test/groups/issue2232/IssueCommandLineTest.java b/testng-jcommander/src/test/java/test/groups/issue2232/IssueCommandLineTest.java new file mode 100644 index 000000000..b505857c4 --- /dev/null +++ b/testng-jcommander/src/test/java/test/groups/issue2232/IssueCommandLineTest.java @@ -0,0 +1,59 @@ +package test.groups.issue2232; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.testng.Reporter; +import org.testng.TestNG; +import org.testng.annotations.Test; +import test.TestHelper; + +/** + * The command line half of {@code test.groups.issue2232.IssueTest}, which stays in {@code + * testng-core} for the in-process case. This is the only test in the build that runs {@code + * org.testng.TestNG} as a real forked process. + */ +public class IssueCommandLineTest { + + private static final int FORK_TIMEOUT_MINUTES = 5; + + @Test(invocationCount = 2, description = "GITHUB-2232") + // Ensuring that the bug doesn't surface even when tests are executed via the command line mode + public void commandlineTest() throws IOException, InterruptedException { + String suitefile = TestHelper.writeSuiteToTempFile(Issue2232Suites.construct()); + List args = Collections.singletonList(suitefile); + int status = exec(Collections.emptyList(), args); + assertThat(status).isEqualTo(0); + } + + private int exec(List jvmArgs, List args) + throws IOException, InterruptedException { + + String javaHome = System.getProperty("java.home"); + String javaBin = javaHome + File.separator + "bin" + File.separator + "java"; + String classpath = System.getProperty("java.class.path"); + String className = TestNG.class.getName(); + List command = new ArrayList<>(); + command.add(javaBin); + command.addAll(jvmArgs); + command.add("-cp"); + command.add(classpath); + command.add(className); + command.addAll(args); + Reporter.log("Executing the command " + command, 2, true); + ProcessBuilder builder = new ProcessBuilder(command); + Process process = builder.inheritIO().start(); + if (!process.waitFor(FORK_TIMEOUT_MINUTES, TimeUnit.MINUTES)) { + process.destroyForcibly(); + throw new AssertionError( + "The forked TestNG run did not finish within " + FORK_TIMEOUT_MINUTES + " minutes"); + } + + return process.exitValue(); + } +} diff --git a/testng-jcommander/src/test/java/test/listeners/ListenerWiringCommandLineTest.java b/testng-jcommander/src/test/java/test/listeners/ListenerWiringCommandLineTest.java new file mode 100644 index 000000000..6b22db345 --- /dev/null +++ b/testng-jcommander/src/test/java/test/listeners/ListenerWiringCommandLineTest.java @@ -0,0 +1,63 @@ +package test.listeners; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.testng.TestNG; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import test.listeners.cliwiring.FirstWiringListener; +import test.listeners.cliwiring.ReverseNameListenerComparator; +import test.listeners.cliwiring.SecondWiringListener; +import test.listeners.cliwiring.WiringLog; +import test.listeners.cliwiring.WiringSampleTest; + +/** + * Covers the {@code -listener} / {@code -listenercomparator} / {@code -usedefaultlisteners} wiring + * end to end. {@code testng-core} exercises listener ordering through the Java API; only the + * command line goes through comma splitting and class loading, so that part is pinned here. + */ +public class ListenerWiringCommandLineTest { + + @BeforeMethod + public void clearLog() { + WiringLog.clear(); + } + + @Test + public void commaSeparatedListenersAreWiredAndOrderedByTheComparator() { + String[] args = { + "-listener", + FirstWiringListener.class.getName() + "," + SecondWiringListener.class.getName(), + "-listenercomparator", + ReverseNameListenerComparator.class.getName(), + "-usedefaultlisteners", + "false", + "-testclass", + WiringSampleTest.class.getName() + }; + + TestNG testng = TestNG.privateMain(args, null); + + assertThat(testng.getStatus()).isZero(); + // Both names made it through the comma split, and the comparator drove the order: + // "Second..." sorts after "First...", and the comparator reverses that. + assertThat(WiringLog.entries()).containsExactly("second", "first"); + } + + @Test + public void semicolonSeparatedListenersAreAlsoAccepted() { + String[] args = { + "-listener", + FirstWiringListener.class.getName() + ";" + SecondWiringListener.class.getName(), + "-usedefaultlisteners", + "false", + "-testclass", + WiringSampleTest.class.getName() + }; + + TestNG testng = TestNG.privateMain(args, null); + + assertThat(testng.getStatus()).isZero(); + assertThat(WiringLog.entries()).containsExactlyInAnyOrder("first", "second"); + } +} diff --git a/testng-jcommander/src/test/java/test/listeners/factory/TestNGFactoryCommandLineTest.java b/testng-jcommander/src/test/java/test/listeners/factory/TestNGFactoryCommandLineTest.java new file mode 100644 index 000000000..7aac69146 --- /dev/null +++ b/testng-jcommander/src/test/java/test/listeners/factory/TestNGFactoryCommandLineTest.java @@ -0,0 +1,31 @@ +package test.listeners.factory; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import org.testng.TestNG; +import org.testng.annotations.Test; +import org.testng.cli.CliOptions; + +/** + * The command line half of {@code test.listeners.factory.TestNGFactoryTest}, which stays in {@code + * testng-core} for the case that drives the Java API. + */ +public class TestNGFactoryCommandLineTest { + + @Test(description = "GITHUB-3059") + public void testListenerFactoryViaConfigurationArg() { + String[] args = + new String[] { + CliOptions.LISTENER_FACTORY, + SampleTestFactory.class.getName(), + CliOptions.TEST_CLASS, + SampleTestCase.class.getName(), + CliOptions.LISTENER, + ExampleListener.class.getName() + }; + TestNG testng = TestNG.privateMain(args, null); + assertThat(SampleTestFactory.instance).isNotNull(); + assertThat(ExampleListener.getInstance()).isNotNull(); + assertThat(testng.getStatus()).isZero(); + } +} diff --git a/testng-core/src/test/java/test/methodselectors/CommandLineTest.java b/testng-jcommander/src/test/java/test/methodselectors/CommandLineTest.java similarity index 75% rename from testng-core/src/test/java/test/methodselectors/CommandLineTest.java rename to testng-jcommander/src/test/java/test/methodselectors/CommandLineTest.java index bbe97e192..c46014a69 100644 --- a/testng-core/src/test/java/test/methodselectors/CommandLineTest.java +++ b/testng-jcommander/src/test/java/test/methodselectors/CommandLineTest.java @@ -1,15 +1,12 @@ package test.methodselectors; -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.List; -import org.testng.ITestResult; import org.testng.TestListenerAdapter; import org.testng.TestNG; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.testng.testhelper.OutputDirectoryPatch; import test.SimpleBaseTest; +import test.TestHelper; public class CommandLineTest extends SimpleBaseTest { @@ -48,8 +45,8 @@ public void commandLineNegativePriorityAllGroups() { TestNG.privateMain(ARG_WITHOUT_GROUPS, tla); String[] passed = {"test1", "test2", "test3"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -58,8 +55,8 @@ public void commandLineNegativePriorityGroup2() { TestNG.privateMain(ARG_WITHOUT_GROUPS, tla); String[] passed = {"test2"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -69,8 +66,8 @@ public void commandLineLessThanPriorityTest1Test() { TestNG.privateMain(ARG_WITH_GROUPS, tla); String[] passed = {"test1", "test2"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -80,8 +77,8 @@ public void commandLineGreaterThanPriorityTest1Test2() { TestNG.privateMain(ARG_WITH_GROUPS, tla); String[] passed = {"test2"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -91,8 +88,8 @@ public void commandLineLessThanPriorityAllTests() { TestNG.privateMain(ARG_WITH_GROUPS, tla); String[] passed = {"test1", "test2", "test3"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -103,8 +100,8 @@ public void commandLineMultipleSelectors() { TestNG.privateMain(ARG_WITH_GROUPS, tla); String[] passed = {"test1", "test2"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -113,8 +110,8 @@ public void commandLineNoTest1Selector() { TestNG.privateMain(ARG_WITHOUT_GROUPS, tla); String[] passed = {"test2", "test3"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test @@ -124,8 +121,8 @@ public void commandLineTestWithXmlFile() { TestNG.privateMain(ARG_WITHOUT_CLASSES, tla); String[] passed = {"test2", "test3"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test(description = "GITHUB-2407") @@ -147,8 +144,8 @@ public void testOverrideExcludedMethodsCommandLineExclusions() { // test1 is excluded, so only test2 is left in the passed list String[] passed = {"test2"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test(description = "GITHUB-2407") @@ -167,8 +164,8 @@ public void testOverrideExcludedMethodsSuiteExclusions() { String[] passed = {}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } @Test(description = "GITHUB-2407") @@ -188,14 +185,7 @@ public void testSpecificTestNamesWithRegexCommandLineExclusions() { String[] passed = {"sampleOutputTest1"}; String[] failed = {}; - verifyTests("Passed", passed, tla.getPassedTests()); - verifyTests("Failed", failed, tla.getFailedTests()); - } - - private void verifyTests(String title, String[] expected, List found) { - - assertThat(found.stream().map(ITestResult::getName).toArray(String[]::new)) - .describedAs(title) - .isEqualTo(expected); + TestHelper.assertPassedTestNames(tla.getPassedTests(), passed); + TestHelper.assertFailedTestNames(tla.getFailedTests(), failed); } } diff --git a/testng-jcommander/src/test/java/test/methodselectors/MethodSelectorInSuiteCommandLineTest.java b/testng-jcommander/src/test/java/test/methodselectors/MethodSelectorInSuiteCommandLineTest.java new file mode 100644 index 000000000..a7c3b4694 --- /dev/null +++ b/testng-jcommander/src/test/java/test/methodselectors/MethodSelectorInSuiteCommandLineTest.java @@ -0,0 +1,36 @@ +package test.methodselectors; + +import org.testng.TestListenerAdapter; +import org.testng.TestNG; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.testng.testhelper.OutputDirectoryPatch; +import test.SimpleBaseTest; +import test.TestHelper; + +/** + * The command line half of {@code test.methodselectors.MethodSelectorInSuiteTest}, which stays in + * {@code testng-core} for the cases that drive the Java API. + */ +public class MethodSelectorInSuiteCommandLineTest extends SimpleBaseTest { + + private TestListenerAdapter m_tla; + + @BeforeMethod + public void setup() { + m_tla = new TestListenerAdapter(); + } + + @Test + public void fileOnCommandLine() { + String[] args = + new String[] { + "-d", + OutputDirectoryPatch.getOutputDirectory(), + getPathToResource("methodselector-in-xml.xml") + }; + TestNG.privateMain(args, m_tla); + + TestHelper.assertPassedTestNames(m_tla.getPassedTests(), "test2"); + } +} diff --git a/testng-core/src/test/java/test/methodselectors/NoTest1MethodSelector.java b/testng-jcommander/src/test/java/test/methodselectors/NoTest1MethodSelector.java similarity index 100% rename from testng-core/src/test/java/test/methodselectors/NoTest1MethodSelector.java rename to testng-jcommander/src/test/java/test/methodselectors/NoTest1MethodSelector.java diff --git a/testng-jcommander/src/test/java/test/reports/EmailableReporterCommandLineTest.java b/testng-jcommander/src/test/java/test/reports/EmailableReporterCommandLineTest.java new file mode 100644 index 000000000..88d17c3ab --- /dev/null +++ b/testng-jcommander/src/test/java/test/reports/EmailableReporterCommandLineTest.java @@ -0,0 +1,71 @@ +package test.reports; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.lang.reflect.Method; +import org.testng.TestNG; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import org.testng.reporters.EmailableReporter2; +import test.SimpleBaseTest; + +/** + * The command line half of {@code test.reports.EmailableReporterTest}, which stays in {@code + * testng-core} for the cases that drive the Java API. These are the only tests covering the {@code + * -reporter} string to {@code ReporterConfig} conversion. + */ +public class EmailableReporterCommandLineTest extends SimpleBaseTest { + + @Test(dataProvider = "getReporterNames", priority = 1) + public void testReportsNameCustomizationViaMainMethodInvocation(String clazzName) { + runTestViaMainMethod(clazzName, null /* no jvm arguments */); + } + + @Test(dataProvider = "getReporterNames", priority = 2) + public void testReportsNameCustomizationViaMainMethodInvocationAndJVMArguments( + String clazzName, String jvm) { + runTestViaMainMethod(clazzName, jvm); + } + + @DataProvider(name = "getReporterNames") + public Object[][] getReporterNames(Method method) { + if (method.getName().toLowerCase().contains("jvmarguments")) { + return new Object[][] {{EmailableReporter2.class.getName(), "emailable.report2.name"}}; + } + return new Object[][] {{EmailableReporter2.class.getName()}}; + } + + private void runTestViaMainMethod(String clazzName, String jvm) { + String name = Long.toString(System.currentTimeMillis()); + File output = createDirInTempDir(name); + String filename = "report" + name + ".html"; + String[] args = { + "-d", + output.getAbsolutePath(), + "-reporter", + clazzName + ":fileName=" + filename, + "src/test/resources/1332.xml" + }; + String previous = jvm == null ? null : System.getProperty(jvm); + try { + if (jvm != null) { + System.setProperty(jvm, filename); + } + TestNG.privateMain(args, null); + } catch (SecurityException t) { + // Gobble Security exception + } finally { + if (jvm != null) { + // Put the property back the way it was, rather than leaving an empty value behind. + if (previous == null) { + System.clearProperty(jvm); + } else { + System.setProperty(jvm, previous); + } + } + } + File actual = new File(output.getAbsolutePath(), filename); + assertThat(actual).exists(); + } +} diff --git a/testng-jcommander/src/test/java/test/thread/CustomExecutorServiceFactoryCommandLineTest.java b/testng-jcommander/src/test/java/test/thread/CustomExecutorServiceFactoryCommandLineTest.java new file mode 100644 index 000000000..9d415cc3e --- /dev/null +++ b/testng-jcommander/src/test/java/test/thread/CustomExecutorServiceFactoryCommandLineTest.java @@ -0,0 +1,82 @@ +package test.thread; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.testng.TestNG; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; +import org.testng.collections.Lists; +import org.testng.xml.XmlSuite; +import test.SimpleBaseTest; +import test.thread.issue3066.Issue3066ExecutorServiceFactory; +import test.thread.issue3066.Issue3066ThreadPoolExecutor; +import test.thread.issue3066.TestClassSample; + +/** + * The command line half of {@code test.thread.CustomExecutorServiceFactoryTest}, which stays in + * {@code testng-core} for the cases that drive the Java API. These are the only tests covering the + * {@code -threadpoolfactoryclass} and {@code -suitethreadpoolsize} options end to end. + */ +public class CustomExecutorServiceFactoryCommandLineTest extends SimpleBaseTest { + + @Test(description = "GITHUB-3066") + public void ensureCanWireInCustomExecutorServiceWhenEnabledViaConfigParam() { + String[] args = { + "-testclass", + TestClassSample.class.getName(), + "-threadpoolfactoryclass", + Issue3066ExecutorServiceFactory.class.getName(), + "-parallel", + "methods" + }; + TestNG.privateMain(args, null); + assertThat(Issue3066ThreadPoolExecutor.isInvoked()).isTrue(); + } + + @Test(description = "GITHUB-3066") + public void ensureCanWireInCustomExecutorServiceWhenEnabledViaConfigForMultipleSuites() { + AtomicInteger counter = new AtomicInteger(1); + List suites = new ArrayList<>(); + File dir = createDirInTempDir("suites"); + Stream.of(TestClassSample.class, TestClassSample.class) + .map( + it -> createXmlSuite("suite-" + counter.get(), "test-" + counter.getAndIncrement(), it)) + .map(XmlSuite::toXml) + .forEach( + it -> { + Path s1 = Paths.get(dir.getAbsolutePath(), UUID.randomUUID() + "-suite.xml"); + try { + Files.writeString(s1, it); + suites.add(s1.toFile().getAbsolutePath()); + } catch (IOException e) { + // Swallowing this would quietly turn a two-suite test into a one-suite one. + throw new UncheckedIOException(e); + } + }); + + List args = + List.of( + "-threadpoolfactoryclass", + Issue3066ExecutorServiceFactory.class.getName(), + "-suitethreadpoolsize", + "2"); + TestNG.privateMain(Lists.merge(suites, args).toArray(String[]::new), null); + assertThat(Issue3066ThreadPoolExecutor.isInvoked()).isTrue(); + } + + @AfterMethod + public void resetState() { + Issue3066ThreadPoolExecutor.resetInvokedState(); + } +} diff --git a/testng-core/src/test/resources/1332.xml b/testng-jcommander/src/test/resources/1332.xml similarity index 100% rename from testng-core/src/test/resources/1332.xml rename to testng-jcommander/src/test/resources/1332.xml diff --git a/testng-jcommander/src/test/resources/methodselector-in-xml.xml b/testng-jcommander/src/test/resources/methodselector-in-xml.xml new file mode 100644 index 000000000..c744137de --- /dev/null +++ b/testng-jcommander/src/test/resources/methodselector-in-xml.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/testng-core/src/test/resources/test/methodselectors/sampleTest.xml b/testng-jcommander/src/test/resources/test/methodselectors/sampleTest.xml similarity index 100% rename from testng-core/src/test/resources/test/methodselectors/sampleTest.xml rename to testng-jcommander/src/test/resources/test/methodselectors/sampleTest.xml diff --git a/testng-core/src/test/resources/test/methodselectors/sampleTestExclusions.xml b/testng-jcommander/src/test/resources/test/methodselectors/sampleTestExclusions.xml similarity index 100% rename from testng-core/src/test/resources/test/methodselectors/sampleTestExclusions.xml rename to testng-jcommander/src/test/resources/test/methodselectors/sampleTestExclusions.xml diff --git a/testng-jcommander/src/test/resources/testnames/main-suite.xml b/testng-jcommander/src/test/resources/testnames/main-suite.xml new file mode 100644 index 000000000..028f96c70 --- /dev/null +++ b/testng-jcommander/src/test/resources/testnames/main-suite.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/testng-core/src/test/resources/testng-configfailure.xml b/testng-jcommander/src/test/resources/testng-configfailure.xml similarity index 100% rename from testng-core/src/test/resources/testng-configfailure.xml rename to testng-jcommander/src/test/resources/testng-configfailure.xml diff --git a/testng-core/src/test/resources/testng-methodselectors.xml b/testng-jcommander/src/test/resources/testng-methodselectors.xml similarity index 100% rename from testng-core/src/test/resources/testng-methodselectors.xml rename to testng-jcommander/src/test/resources/testng-methodselectors.xml diff --git a/testng-jcommander/testng-jcommander-build.gradle.kts b/testng-jcommander/testng-jcommander-build.gradle.kts new file mode 100644 index 000000000..11e5359e8 --- /dev/null +++ b/testng-jcommander/testng-jcommander-build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + id("testng.java-library") +} + +description = "JCommander based command line front end for TestNG" + +dependencies { + api(projects.testngCli) + api("org.jcommander:jcommander:2.0") + // org.testng.internal.Yaml, used by Converter, exposes snakeyaml on its API + compileOnly("org.yaml:snakeyaml:2.2") + testImplementation(projects.testngTestKit) +} diff --git a/testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest1.java b/testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest1.java similarity index 100% rename from testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest1.java rename to testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest1.java diff --git a/testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest2.java b/testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest2.java similarity index 100% rename from testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest2.java rename to testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest2.java diff --git a/testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest3.java b/testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest3.java similarity index 100% rename from testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest3.java rename to testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest3.java diff --git a/testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest4.java b/testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest4.java similarity index 100% rename from testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest4.java rename to testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest4.java diff --git a/testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest5.java b/testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest5.java similarity index 100% rename from testng-core/src/test/java/org/testng/jarfileutils/org/testng/SampleTest5.java rename to testng-test-kit/src/main/java/org/testng/jarfileutils/org/testng/SampleTest5.java diff --git a/testng-core/src/test/java/org/testng/testhelper/JarCreator.java b/testng-test-kit/src/main/java/org/testng/testhelper/JarCreator.java similarity index 90% rename from testng-core/src/test/java/org/testng/testhelper/JarCreator.java rename to testng-test-kit/src/main/java/org/testng/testhelper/JarCreator.java index d75b3a38b..909ee3eb3 100644 --- a/testng-core/src/test/java/org/testng/testhelper/JarCreator.java +++ b/testng-test-kit/src/main/java/org/testng/testhelper/JarCreator.java @@ -28,6 +28,9 @@ public static File generateJar( Class[] classes, String[] resources, String prefix, String archiveName) throws IOException { File jarFile = File.createTempFile(prefix, ".jar"); + // Every generated jar goes through here, so scheduling the deletion once covers all callers, + // including any added later that would forget to clean up in a teardown. + jarFile.deleteOnExit(); JavaArchive archive = ShrinkWrap.create(JavaArchive.class, archiveName).addClasses(classes); for (String resource : resources) { archive = archive.addAsResource(resource); diff --git a/testng-core/src/test/java/org/testng/testhelper/OutputDirectoryPatch.java b/testng-test-kit/src/main/java/org/testng/testhelper/OutputDirectoryPatch.java similarity index 100% rename from testng-core/src/test/java/org/testng/testhelper/OutputDirectoryPatch.java rename to testng-test-kit/src/main/java/org/testng/testhelper/OutputDirectoryPatch.java diff --git a/testng-core/src/test/java/test/InvokedMethodNameListener.java b/testng-test-kit/src/main/java/test/InvokedMethodNameListener.java similarity index 95% rename from testng-core/src/test/java/test/InvokedMethodNameListener.java rename to testng-test-kit/src/main/java/test/InvokedMethodNameListener.java index 2909b87de..4ec50744c 100644 --- a/testng-core/src/test/java/test/InvokedMethodNameListener.java +++ b/testng-test-kit/src/main/java/test/InvokedMethodNameListener.java @@ -1,11 +1,11 @@ package test; -import com.google.common.base.Joiner; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; @@ -141,11 +141,7 @@ private static String getName(ITestResult result) { name = testName + "#" + methodName; } if (result.getParameters().length != 0) { - name = - name - + "(" - + Joiner.on(",").useForNull("null").join(getParameterNames(result.getParameters())) - + ")"; + name = name + "(" + String.join(",", getParameterNames(result.getParameters())) + ")"; } return name; } @@ -157,7 +153,11 @@ private static List getParameterNames(Object[] parameters) { result.add("null"); } else { if (parameter instanceof Object[]) { - result.add("[" + Joiner.on(",").useForNull("null").join((Object[]) parameter) + "]"); + StringJoiner joined = new StringJoiner(","); + for (Object item : (Object[]) parameter) { + joined.add(String.valueOf(item)); + } + result.add("[" + joined + "]"); } else { result.add(parameter.toString()); } diff --git a/testng-test-kit/src/main/java/test/TestHelper.java b/testng-test-kit/src/main/java/test/TestHelper.java new file mode 100644 index 000000000..62cea4508 --- /dev/null +++ b/testng-test-kit/src/main/java/test/TestHelper.java @@ -0,0 +1,132 @@ +package test; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import org.testng.ITestResult; +import org.testng.TestListenerAdapter; +import org.testng.TestNG; +import org.testng.xml.XmlClass; +import org.testng.xml.XmlSuite; +import org.testng.xml.XmlTest; + +public class TestHelper { + + /** + * Writes a suite to a temporary file and returns its path, the way a command line test needs it. + * + * @param suite the suite to serialize. + * @return the absolute path of the written file. + */ + public static String writeSuiteToTempFile(XmlSuite suite) throws IOException { + return writeSuiteToTempFile(suite.toXml()); + } + + /** + * @param suiteXml the already serialized suite. + * @return the absolute path of the written file. + */ + public static String writeSuiteToTempFile(String suiteXml) throws IOException { + Path file = Files.createTempFile("testng", ".xml"); + file.toFile().deleteOnExit(); + Files.write(file, suiteXml.getBytes(StandardCharsets.UTF_8)); + return file.toFile().getAbsolutePath(); + } + + /** + * Asserts that exactly the given test methods passed, in order. + * + * @param found the results collected by a listener. + * @param expected the expected method names. + */ + public static void assertPassedTestNames(List found, String... expected) { + assertTestNames("passed tests", found, expected); + } + + /** + * Asserts that exactly the given test methods failed, in order. + * + * @param found the results collected by a listener. + * @param expected the expected method names. + */ + public static void assertFailedTestNames(List found, String... expected) { + assertTestNames("failed tests", found, expected); + } + + private static void assertTestNames( + String description, List found, String... expected) { + assertThat(found.stream().map(ITestResult::getName).toArray(String[]::new)) + .describedAs(description) + .isEqualTo(expected); + } + + /** + * Asserts the configuration and skip counters a run produced. + * + * @param tla the listener that observed the run. + */ + public static void assertCounts( + TestListenerAdapter tla, + int configurationFailures, + int configurationSkips, + int skippedTests) { + assertThat(tla.getConfigurationFailures()) + .describedAs("configuration failures") + .hasSize(configurationFailures); + assertThat(tla.getConfigurationSkips()) + .describedAs("configuration skips") + .hasSize(configurationSkips); + assertThat(tla.getSkippedTests()).describedAs("skipped tests").hasSize(skippedTests); + } + + /* + * TestNG issues a warning if the XML misses DOCTYPE, so here's a common header for + * xml suites generated in the tests. + */ + public static final String SUITE_XML_HEADER = + "\n" + + "\n"; + + public static XmlSuite createSuite(String cls, String suiteName) { + XmlSuite result = new XmlSuite(); + result.setName(suiteName); + + XmlTest test = new XmlTest(result); + test.setName("TmpTest"); + test.setXmlClasses(Collections.singletonList(new XmlClass(cls))); + + return result; + } + + public static TestNG createTestNG() throws IOException { + return createTestNG(createRandomDirectory()); + } + + public static TestNG createTestNG(XmlSuite suite) throws IOException { + return createTestNG(suite, createRandomDirectory()); + } + + public static TestNG createTestNG(XmlSuite suite, Path outputDir) { + TestNG result = createTestNG(outputDir); + result.setXmlSuites(Collections.singletonList(suite)); + return result; + } + + private static TestNG createTestNG(Path outputDir) { + TestNG result = new TestNG(); + result.setOutputDirectory(outputDir.toAbsolutePath().toString()); + + return result; + } + + public static Path createRandomDirectory() throws IOException { + Path directory = Files.createTempDirectory("testng-tmp-"); + directory.toFile().deleteOnExit(); + return directory; + } +} diff --git a/testng-core/src/test/java/test/commandline/issue341/LocalLogAggregator.java b/testng-test-kit/src/main/java/test/commandline/issue341/LocalLogAggregator.java similarity index 63% rename from testng-core/src/test/java/test/commandline/issue341/LocalLogAggregator.java rename to testng-test-kit/src/main/java/test/commandline/issue341/LocalLogAggregator.java index bd045ba43..4bfe28c79 100644 --- a/testng-core/src/test/java/test/commandline/issue341/LocalLogAggregator.java +++ b/testng-test-kit/src/main/java/test/commandline/issue341/LocalLogAggregator.java @@ -1,6 +1,7 @@ package test.commandline.issue341; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; @@ -16,7 +17,16 @@ public void afterInvocation(IInvokedMethod method, ITestResult testResult) { logs.addAll(Reporter.getOutput(testResult)); } + /** + * @return a snapshot, so that a caller cannot observe or mutate the live collection while a run + * is still writing to it. + */ public static Set getLogs() { - return logs; + return new HashSet<>(logs); + } + + /** Static state survives the JVM, so a test that asserts on a count has to start from empty. */ + public static void clearLogs() { + logs.clear(); } } diff --git a/testng-core/src/test/java/test/commandline/issue341/TestSampleA.java b/testng-test-kit/src/main/java/test/commandline/issue341/TestSampleA.java similarity index 100% rename from testng-core/src/test/java/test/commandline/issue341/TestSampleA.java rename to testng-test-kit/src/main/java/test/commandline/issue341/TestSampleA.java diff --git a/testng-core/src/test/java/test/commandline/issue341/TestSampleB.java b/testng-test-kit/src/main/java/test/commandline/issue341/TestSampleB.java similarity index 100% rename from testng-core/src/test/java/test/commandline/issue341/TestSampleB.java rename to testng-test-kit/src/main/java/test/commandline/issue341/TestSampleB.java diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/ClassWithFailedBeforeMethodAndMultipleTests.java b/testng-test-kit/src/main/java/test/configurationfailurepolicy/ClassWithFailedBeforeMethodAndMultipleTests.java similarity index 100% rename from testng-core/src/test/java/test/configurationfailurepolicy/ClassWithFailedBeforeMethodAndMultipleTests.java rename to testng-test-kit/src/main/java/test/configurationfailurepolicy/ClassWithFailedBeforeMethodAndMultipleTests.java diff --git a/testng-test-kit/src/main/java/test/groups/issue2232/Issue2232Suites.java b/testng-test-kit/src/main/java/test/groups/issue2232/Issue2232Suites.java new file mode 100644 index 000000000..62b5849b9 --- /dev/null +++ b/testng-test-kit/src/main/java/test/groups/issue2232/Issue2232Suites.java @@ -0,0 +1,43 @@ +package test.groups.issue2232; + +import java.util.Collections; +import org.testng.xml.XmlGroups; +import org.testng.xml.XmlPackage; +import org.testng.xml.XmlRun; +import org.testng.xml.XmlSuite; +import org.testng.xml.XmlTest; + +/** + * The GITHUB-2232 suite, shared by the in-process test in {@code testng-core} and the forked one in + * {@code testng-jcommander}. The scanned package is spelled out rather than derived from the + * caller, so the two halves cannot drift apart by moving. + */ +public final class Issue2232Suites { + + private static final String SAMPLES_PACKAGE = "test.groups.issue2232.samples.*"; + + private Issue2232Suites() {} + + public static XmlSuite construct() { + XmlSuite xmlsuite = new XmlSuite(); + xmlsuite.setName("2232_suite"); + xmlsuite.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); + xmlsuite.setThreadCount(256); + xmlsuite.setParallel(XmlSuite.ParallelMode.CLASSES); + + XmlTest xmltest = new XmlTest(xmlsuite); + xmltest.setName("2232_test"); + + XmlRun xmlrun = new XmlRun(); + xmlrun.onInclude("Group2"); + xmlrun.onExclude("Broken"); + XmlGroups xmlgroup = new XmlGroups(); + xmlgroup.setRun(xmlrun); + xmltest.setGroups(xmlgroup); + + XmlPackage xmlpackage = new XmlPackage(); + xmlpackage.setName(SAMPLES_PACKAGE); + xmltest.setPackages(Collections.singletonList(xmlpackage)); + return xmlsuite; + } +} diff --git a/testng-core/src/test/java/test/groups/issue2232/samples/SampleTest.java b/testng-test-kit/src/main/java/test/groups/issue2232/samples/SampleTest.java similarity index 100% rename from testng-core/src/test/java/test/groups/issue2232/samples/SampleTest.java rename to testng-test-kit/src/main/java/test/groups/issue2232/samples/SampleTest.java diff --git a/testng-core/src/test/java/test/groups/issue2232/samples/SampleTest2.java b/testng-test-kit/src/main/java/test/groups/issue2232/samples/SampleTest2.java similarity index 100% rename from testng-core/src/test/java/test/groups/issue2232/samples/SampleTest2.java rename to testng-test-kit/src/main/java/test/groups/issue2232/samples/SampleTest2.java diff --git a/testng-test-kit/src/main/java/test/listeners/cliwiring/FirstWiringListener.java b/testng-test-kit/src/main/java/test/listeners/cliwiring/FirstWiringListener.java new file mode 100644 index 000000000..59f9d4deb --- /dev/null +++ b/testng-test-kit/src/main/java/test/listeners/cliwiring/FirstWiringListener.java @@ -0,0 +1,10 @@ +package test.listeners.cliwiring; + +import org.testng.IExecutionListener; + +public class FirstWiringListener implements IExecutionListener { + @Override + public void onExecutionStart() { + WiringLog.record("first"); + } +} diff --git a/testng-test-kit/src/main/java/test/listeners/cliwiring/ReverseNameListenerComparator.java b/testng-test-kit/src/main/java/test/listeners/cliwiring/ReverseNameListenerComparator.java new file mode 100644 index 000000000..09db748f6 --- /dev/null +++ b/testng-test-kit/src/main/java/test/listeners/cliwiring/ReverseNameListenerComparator.java @@ -0,0 +1,12 @@ +package test.listeners.cliwiring; + +import org.testng.ITestNGListener; +import org.testng.ListenerComparator; + +/** Orders listeners by class name, descending, so the wiring order is observable. */ +public class ReverseNameListenerComparator implements ListenerComparator { + @Override + public int compare(ITestNGListener first, ITestNGListener second) { + return second.getClass().getName().compareTo(first.getClass().getName()); + } +} diff --git a/testng-test-kit/src/main/java/test/listeners/cliwiring/SecondWiringListener.java b/testng-test-kit/src/main/java/test/listeners/cliwiring/SecondWiringListener.java new file mode 100644 index 000000000..a900dd09d --- /dev/null +++ b/testng-test-kit/src/main/java/test/listeners/cliwiring/SecondWiringListener.java @@ -0,0 +1,10 @@ +package test.listeners.cliwiring; + +import org.testng.IExecutionListener; + +public class SecondWiringListener implements IExecutionListener { + @Override + public void onExecutionStart() { + WiringLog.record("second"); + } +} diff --git a/testng-test-kit/src/main/java/test/listeners/cliwiring/WiringLog.java b/testng-test-kit/src/main/java/test/listeners/cliwiring/WiringLog.java new file mode 100644 index 000000000..83a4d1ed5 --- /dev/null +++ b/testng-test-kit/src/main/java/test/listeners/cliwiring/WiringLog.java @@ -0,0 +1,24 @@ +package test.listeners.cliwiring; + +import java.util.List; +import org.testng.collections.Lists; + +/** Records the order in which the sample listeners below were invoked. */ +public final class WiringLog { + + private static final List ENTRIES = Lists.newArrayList(); + + private WiringLog() {} + + public static void record(String entry) { + ENTRIES.add(entry); + } + + public static List entries() { + return Lists.newArrayList(ENTRIES); + } + + public static void clear() { + ENTRIES.clear(); + } +} diff --git a/testng-test-kit/src/main/java/test/listeners/cliwiring/WiringSampleTest.java b/testng-test-kit/src/main/java/test/listeners/cliwiring/WiringSampleTest.java new file mode 100644 index 000000000..70077e545 --- /dev/null +++ b/testng-test-kit/src/main/java/test/listeners/cliwiring/WiringSampleTest.java @@ -0,0 +1,8 @@ +package test.listeners.cliwiring; + +import org.testng.annotations.Test; + +public class WiringSampleTest { + @Test + public void sample() {} +} diff --git a/testng-core/src/test/java/test/listeners/factory/ExampleListener.java b/testng-test-kit/src/main/java/test/listeners/factory/ExampleListener.java similarity index 100% rename from testng-core/src/test/java/test/listeners/factory/ExampleListener.java rename to testng-test-kit/src/main/java/test/listeners/factory/ExampleListener.java diff --git a/testng-core/src/test/java/test/listeners/factory/SampleTestCase.java b/testng-test-kit/src/main/java/test/listeners/factory/SampleTestCase.java similarity index 100% rename from testng-core/src/test/java/test/listeners/factory/SampleTestCase.java rename to testng-test-kit/src/main/java/test/listeners/factory/SampleTestCase.java diff --git a/testng-core/src/test/java/test/listeners/factory/SampleTestFactory.java b/testng-test-kit/src/main/java/test/listeners/factory/SampleTestFactory.java similarity index 100% rename from testng-core/src/test/java/test/listeners/factory/SampleTestFactory.java rename to testng-test-kit/src/main/java/test/listeners/factory/SampleTestFactory.java diff --git a/testng-core/src/test/java/test/methodselectors/AllTestsMethodSelector.java b/testng-test-kit/src/main/java/test/methodselectors/AllTestsMethodSelector.java similarity index 100% rename from testng-core/src/test/java/test/methodselectors/AllTestsMethodSelector.java rename to testng-test-kit/src/main/java/test/methodselectors/AllTestsMethodSelector.java diff --git a/testng-core/src/test/java/test/methodselectors/NoTest.java b/testng-test-kit/src/main/java/test/methodselectors/NoTest.java similarity index 100% rename from testng-core/src/test/java/test/methodselectors/NoTest.java rename to testng-test-kit/src/main/java/test/methodselectors/NoTest.java diff --git a/testng-core/src/test/java/test/methodselectors/NoTestSelector.java b/testng-test-kit/src/main/java/test/methodselectors/NoTestSelector.java similarity index 100% rename from testng-core/src/test/java/test/methodselectors/NoTestSelector.java rename to testng-test-kit/src/main/java/test/methodselectors/NoTestSelector.java diff --git a/testng-core/src/test/java/test/methodselectors/SampleTest.java b/testng-test-kit/src/main/java/test/methodselectors/SampleTest.java similarity index 100% rename from testng-core/src/test/java/test/methodselectors/SampleTest.java rename to testng-test-kit/src/main/java/test/methodselectors/SampleTest.java diff --git a/testng-core/src/test/java/test/methodselectors/Test2MethodSelector.java b/testng-test-kit/src/main/java/test/methodselectors/Test2MethodSelector.java similarity index 94% rename from testng-core/src/test/java/test/methodselectors/Test2MethodSelector.java rename to testng-test-kit/src/main/java/test/methodselectors/Test2MethodSelector.java index 5bf196e21..e0c7ddf26 100644 --- a/testng-core/src/test/java/test/methodselectors/Test2MethodSelector.java +++ b/testng-test-kit/src/main/java/test/methodselectors/Test2MethodSelector.java @@ -11,7 +11,7 @@ public class Test2MethodSelector implements IMethodSelector { public boolean includeMethod( IMethodSelectorContext context, ITestNGMethod method, boolean isTestMethod) { for (String group : method.getGroups()) { - if (group.equals("test2")) { + if ("test2".equals(group)) { context.setStopped(true); return true; } diff --git a/testng-core/src/test/java/test/reports/ReporterSample.java b/testng-test-kit/src/main/java/test/reports/ReporterSample.java similarity index 100% rename from testng-core/src/test/java/test/reports/ReporterSample.java rename to testng-test-kit/src/main/java/test/reports/ReporterSample.java diff --git a/testng-core/src/test/java/test/testnames/TestNamesFeature.java b/testng-test-kit/src/main/java/test/testnames/TestNamesFeature.java similarity index 100% rename from testng-core/src/test/java/test/testnames/TestNamesFeature.java rename to testng-test-kit/src/main/java/test/testnames/TestNamesFeature.java diff --git a/testng-core/src/test/java/test/thread/issue3066/Issue3066ExecutorServiceFactory.java b/testng-test-kit/src/main/java/test/thread/issue3066/Issue3066ExecutorServiceFactory.java similarity index 100% rename from testng-core/src/test/java/test/thread/issue3066/Issue3066ExecutorServiceFactory.java rename to testng-test-kit/src/main/java/test/thread/issue3066/Issue3066ExecutorServiceFactory.java diff --git a/testng-core/src/test/java/test/thread/issue3066/Issue3066ThreadPoolExecutor.java b/testng-test-kit/src/main/java/test/thread/issue3066/Issue3066ThreadPoolExecutor.java similarity index 100% rename from testng-core/src/test/java/test/thread/issue3066/Issue3066ThreadPoolExecutor.java rename to testng-test-kit/src/main/java/test/thread/issue3066/Issue3066ThreadPoolExecutor.java diff --git a/testng-core/src/test/java/test/thread/issue3066/TestClassSample.java b/testng-test-kit/src/main/java/test/thread/issue3066/TestClassSample.java similarity index 100% rename from testng-core/src/test/java/test/thread/issue3066/TestClassSample.java rename to testng-test-kit/src/main/java/test/thread/issue3066/TestClassSample.java diff --git a/testng-core/src/test/kotlin/test/SimpleBaseTest.kt b/testng-test-kit/src/main/kotlin/test/SimpleBaseTest.kt similarity index 99% rename from testng-core/src/test/kotlin/test/SimpleBaseTest.kt rename to testng-test-kit/src/main/kotlin/test/SimpleBaseTest.kt index 5651c2647..67e589c4c 100644 --- a/testng-core/src/test/kotlin/test/SimpleBaseTest.kt +++ b/testng-test-kit/src/main/kotlin/test/SimpleBaseTest.kt @@ -342,7 +342,7 @@ open class SimpleBaseTest { fileName: File, regexp: String, resultLines: MutableList - ) = grep(FileReader(fileName), regexp, resultLines) + ) = FileReader(fileName).use { grep(it, regexp, resultLines) } @JvmStatic protected fun grep( diff --git a/testng-core/src/test/resources/jarfileutils/child.xml b/testng-test-kit/src/main/resources/jarfileutils/child.xml similarity index 100% rename from testng-core/src/test/resources/jarfileutils/child.xml rename to testng-test-kit/src/main/resources/jarfileutils/child.xml diff --git a/testng-core/src/test/resources/jarfileutils/child/child.xml b/testng-test-kit/src/main/resources/jarfileutils/child/child.xml similarity index 100% rename from testng-core/src/test/resources/jarfileutils/child/child.xml rename to testng-test-kit/src/main/resources/jarfileutils/child/child.xml diff --git a/testng-core/src/test/resources/jarfileutils/child/childofchild/childofchild.xml b/testng-test-kit/src/main/resources/jarfileutils/child/childofchild/childofchild.xml similarity index 100% rename from testng-core/src/test/resources/jarfileutils/child/childofchild/childofchild.xml rename to testng-test-kit/src/main/resources/jarfileutils/child/childofchild/childofchild.xml diff --git a/testng-core/src/test/resources/jarfileutils/childofchild/childofchild.xml b/testng-test-kit/src/main/resources/jarfileutils/childofchild/childofchild.xml similarity index 100% rename from testng-core/src/test/resources/jarfileutils/childofchild/childofchild.xml rename to testng-test-kit/src/main/resources/jarfileutils/childofchild/childofchild.xml diff --git a/testng-core/src/test/resources/jarfileutils/testng-tests.xml b/testng-test-kit/src/main/resources/jarfileutils/testng-tests.xml similarity index 100% rename from testng-core/src/test/resources/jarfileutils/testng-tests.xml rename to testng-test-kit/src/main/resources/jarfileutils/testng-tests.xml diff --git a/testng-test-kit/testng-test-kit-build.gradle.kts b/testng-test-kit/testng-test-kit-build.gradle.kts index 0438ec3bf..b214fd14c 100644 --- a/testng-test-kit/testng-test-kit-build.gradle.kts +++ b/testng-test-kit/testng-test-kit-build.gradle.kts @@ -1,3 +1,25 @@ plugins { - id("testng.java-library") + id("testng.kotlin-library") +} + +description = "Test fixtures shared by the TestNG modules. Never published." + +dependencies { + api(platform("org.jetbrains.kotlin:kotlin-bom:2.4.10")) + api("org.jetbrains.kotlin:kotlin-stdlib") { + because("SimpleBaseTest is a Kotlin class extended from Java-only modules, so the stdlib is part of this module's ABI, not an implementation detail") + } + + // compileOnly on purpose: this module is only ever put on a test classpath that already + // carries TestNG, and an api edge here would drag testng-core into modules that deliberately + // resolve a released org.testng artifact instead. + compileOnly(projects.testngCore) + + api("org.assertj:assertj-core:3.27.7") { + because("SimpleBaseTest exposes assertions") + } + api("org.jboss.shrinkwrap:shrinkwrap-api:1.2.6") { + because("JarCreator builds test jars") + } + runtimeOnly("org.jboss.shrinkwrap:shrinkwrap-impl-base:1.2.6") } diff --git a/testng-test-osgi/src/test/java/org/testng/test/osgi/PlainOsgiTest.java b/testng-test-osgi/src/test/java/org/testng/test/osgi/PlainOsgiTest.java index 7bac96e99..36a0716a1 100644 --- a/testng-test-osgi/src/test/java/org/testng/test/osgi/PlainOsgiTest.java +++ b/testng-test-osgi/src/test/java/org/testng/test/osgi/PlainOsgiTest.java @@ -6,12 +6,14 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.util.ServiceLoader; import org.ops4j.pax.exam.Configuration; import org.ops4j.pax.exam.Option; import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; import org.ops4j.pax.exam.spi.reactors.PerMethod; import org.ops4j.pax.exam.testng.listener.PaxExam; import org.testng.IModuleFactory; +import org.testng.ITestNGCliRunner; import org.testng.TestNG; import org.testng.annotations.Listeners; import org.testng.annotations.Test; @@ -46,8 +48,20 @@ public void guiceModuleFactoryLoads() { assertThat(IModuleFactory.class.getMethods()).isNotEmpty(); } + /** + * The command line front end is discovered through {@link java.util.ServiceLoader}. Inside the + * merged bundle the provider sits next to the SPI, so looking it up with the classloader that + * owns the interface has to work without any SPI-Fly weaving. + */ @Test - public void jcommanderLoads() { + public void cliRunnerServiceLoads() { + ServiceLoader loader = + ServiceLoader.load(ITestNGCliRunner.class, ITestNGCliRunner.class.getClassLoader()); + // next(), not just hasNext(): hasNext only resolves the class, while instantiating the provider + // is what proves the bundle's mandatory com.beust.jcommander import is actually wired. + ITestNGCliRunner runner = loader.iterator().next(); + assertThat(runner).isNotNull(); + // TestNG has the widest transitive linkage in the bundle; keep forcing it to load. assertThat(TestNG.class.getFields()).isNotEmpty(); } diff --git a/testng/testng-build.gradle.kts b/testng/testng-build.gradle.kts index d8c2cbfe3..8a64c1f90 100644 --- a/testng/testng-build.gradle.kts +++ b/testng/testng-build.gradle.kts @@ -29,9 +29,13 @@ dependencies { // would be selected automatically shadedDependencyElements("org.testng:testng-asserts:1.0.0") shadedDependencyElements(projects.testngCore) + // Brings testng-cli as well, so that `java -cp testng.jar org.testng.TestNG` keeps working + shadedDependencyElements(projects.testngJcommander) } tasks.mergedJar { + // No two modules declare the same service today; this future-proofs the merge if they ever do + mergeServiceFiles() manifest { // providers.gradleProperty does not work // see https://github.com/gradle/gradle/issues/14972 @@ -63,6 +67,8 @@ tasks.mergedJar { "Export-Package" to """ org.testng org.testng.annotations + org.testng.cli + org.testng.cli.jcommander org.testng.collections org.testng.internal org.testng.internal.annotations