View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.booterclient;
20  
21  import javax.annotation.Nonnull;
22  
23  import java.io.Closeable;
24  import java.io.File;
25  import java.io.IOException;
26  import java.util.Collection;
27  import java.util.Map;
28  import java.util.Queue;
29  import java.util.Set;
30  import java.util.concurrent.ArrayBlockingQueue;
31  import java.util.concurrent.Callable;
32  import java.util.concurrent.ConcurrentLinkedQueue;
33  import java.util.concurrent.ConcurrentSkipListSet;
34  import java.util.concurrent.ExecutionException;
35  import java.util.concurrent.ExecutorService;
36  import java.util.concurrent.Future;
37  import java.util.concurrent.LinkedBlockingQueue;
38  import java.util.concurrent.ScheduledExecutorService;
39  import java.util.concurrent.ScheduledFuture;
40  import java.util.concurrent.ThreadFactory;
41  import java.util.concurrent.ThreadPoolExecutor;
42  import java.util.concurrent.atomic.AtomicInteger;
43  
44  import org.apache.maven.plugin.surefire.CommonReflector;
45  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
46  import org.apache.maven.plugin.surefire.SurefireProperties;
47  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.AbstractCommandReader;
48  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.Commandline;
49  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.NotifiableTestStream;
50  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.TestLessInputStream;
51  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.TestProvidingInputStream;
52  import org.apache.maven.plugin.surefire.booterclient.output.ForkClient;
53  import org.apache.maven.plugin.surefire.booterclient.output.InPluginProcessDumpSingleton;
54  import org.apache.maven.plugin.surefire.booterclient.output.NativeStdErrStreamConsumer;
55  import org.apache.maven.plugin.surefire.booterclient.output.ThreadedStreamConsumer;
56  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
57  import org.apache.maven.plugin.surefire.report.DefaultReporterFactory;
58  import org.apache.maven.plugin.surefire.report.ReportsMerger;
59  import org.apache.maven.surefire.api.booter.Shutdown;
60  import org.apache.maven.surefire.api.fork.ForkNodeArguments;
61  import org.apache.maven.surefire.api.provider.SurefireProvider;
62  import org.apache.maven.surefire.api.report.StackTraceWriter;
63  import org.apache.maven.surefire.api.suite.RunResult;
64  import org.apache.maven.surefire.api.testset.TestRequest;
65  import org.apache.maven.surefire.api.util.DefaultScanResult;
66  import org.apache.maven.surefire.booter.AbstractPathConfiguration;
67  import org.apache.maven.surefire.booter.PropertiesWrapper;
68  import org.apache.maven.surefire.booter.ProviderConfiguration;
69  import org.apache.maven.surefire.booter.ProviderFactory;
70  import org.apache.maven.surefire.booter.StartupConfiguration;
71  import org.apache.maven.surefire.booter.SurefireBooterForkException;
72  import org.apache.maven.surefire.booter.SurefireExecutionException;
73  import org.apache.maven.surefire.extensions.EventHandler;
74  import org.apache.maven.surefire.extensions.ForkChannel;
75  import org.apache.maven.surefire.extensions.ForkNodeFactory;
76  import org.apache.maven.surefire.extensions.Stoppable;
77  import org.apache.maven.surefire.extensions.util.CommandlineExecutor;
78  import org.apache.maven.surefire.extensions.util.CommandlineStreams;
79  import org.apache.maven.surefire.extensions.util.CountdownCloseable;
80  import org.apache.maven.surefire.extensions.util.LineConsumerThread;
81  
82  import static java.lang.StrictMath.min;
83  import static java.lang.System.currentTimeMillis;
84  import static java.lang.Thread.currentThread;
85  import static java.util.Collections.addAll;
86  import static java.util.UUID.randomUUID;
87  import static java.util.concurrent.Executors.newScheduledThreadPool;
88  import static java.util.concurrent.TimeUnit.MILLISECONDS;
89  import static java.util.concurrent.TimeUnit.SECONDS;
90  import static java.util.stream.Collectors.toList;
91  import static java.util.stream.StreamSupport.stream;
92  import static org.apache.maven.plugin.surefire.AbstractSurefireMojo.createCopyAndReplaceForkNumPlaceholder;
93  import static org.apache.maven.plugin.surefire.SurefireHelper.DUMP_FILE_PREFIX;
94  import static org.apache.maven.plugin.surefire.SurefireHelper.replaceForkThreadsInPath;
95  import static org.apache.maven.plugin.surefire.booterclient.ForkNumberBucket.drawNumber;
96  import static org.apache.maven.plugin.surefire.booterclient.ForkNumberBucket.returnNumber;
97  import static org.apache.maven.plugin.surefire.booterclient.lazytestprovider.TestLessInputStream.TestLessInputStreamBuilder;
98  import static org.apache.maven.surefire.api.cli.CommandLineOption.SHOW_ERRORS;
99  import static org.apache.maven.surefire.api.suite.RunResult.SUCCESS;
100 import static org.apache.maven.surefire.api.suite.RunResult.failure;
101 import static org.apache.maven.surefire.api.suite.RunResult.timeout;
102 import static org.apache.maven.surefire.api.util.internal.ConcurrencyUtils.runIfZeroCountDown;
103 import static org.apache.maven.surefire.api.util.internal.DaemonThreadFactory.newDaemonThread;
104 import static org.apache.maven.surefire.api.util.internal.DaemonThreadFactory.newDaemonThreadFactory;
105 import static org.apache.maven.surefire.api.util.internal.StringUtils.NL;
106 import static org.apache.maven.surefire.booter.SystemPropertyManager.writePropertiesFile;
107 import static org.apache.maven.surefire.shared.utils.cli.ShutdownHookUtils.addShutDownHook;
108 import static org.apache.maven.surefire.shared.utils.cli.ShutdownHookUtils.removeShutdownHook;
109 
110 /**
111  * Starts the fork or runs in-process.
112  * <br>
113  * Lives only on the plugin-side (not present in remote vms)
114  * <br>
115  * Knows how to fork new vms and also how to delegate non-forking invocation to SurefireStarter directly
116  *
117  * @author Jason van Zyl
118  * @author Emmanuel Venisse
119  * @author Brett Porter
120  * @author Dan Fabulich
121  * @author Carlos Sanchez
122  * @author Kristian Rosenvold
123  */
124 public class ForkStarter {
125     private static final String EXECUTION_EXCEPTION = "ExecutionException";
126 
127     private static final long PING_IN_SECONDS = 10;
128 
129     private static final int TIMEOUT_CHECK_PERIOD_MILLIS = 100;
130 
131     private static final ThreadFactory FORKED_JVM_DAEMON_THREAD_FACTORY =
132             newDaemonThreadFactory("surefire-fork-starter");
133 
134     private static final ThreadFactory SHUTDOWN_HOOK_THREAD_FACTORY =
135             newDaemonThreadFactory("surefire-jvm-killer-shutdownhook");
136 
137     private static final AtomicInteger SYSTEM_PROPERTIES_FILE_COUNTER = new AtomicInteger();
138 
139     private final Set<String> logsAtEnd = new ConcurrentSkipListSet<>();
140 
141     private final ScheduledExecutorService pingThreadScheduler = createPingScheduler();
142 
143     private final ScheduledExecutorService timeoutCheckScheduler;
144 
145     private final Queue<ForkClient> currentForkClients;
146 
147     private final int forkedProcessTimeoutInSeconds;
148 
149     private final ProviderConfiguration providerConfiguration;
150 
151     private final StartupConfiguration startupConfiguration;
152 
153     private final ForkConfiguration forkConfiguration;
154 
155     private final StartupReportConfiguration startupReportConfiguration;
156 
157     private final ConsoleLogger log;
158 
159     private final ReportsMerger reportMerger;
160 
161     private final Collection<DefaultReporterFactory> defaultReporterFactories;
162 
163     /**
164      * Closes stuff, with a shutdown hook to make sure things really get closed.
165      */
166     private final class CloseableCloser implements Runnable, Closeable {
167         private final int jvmRun;
168 
169         private final Queue<Closeable> testProvidingInputStream;
170 
171         private final Thread inputStreamCloserHook;
172 
173         CloseableCloser(int jvmRun, Closeable... testProvidingInputStream) {
174             this.jvmRun = jvmRun;
175             this.testProvidingInputStream = new ConcurrentLinkedQueue<>();
176             addAll(this.testProvidingInputStream, testProvidingInputStream);
177             if (this.testProvidingInputStream.isEmpty()) {
178                 inputStreamCloserHook = null;
179             } else {
180                 inputStreamCloserHook = newDaemonThread(this, "closer-shutdown-hook");
181                 addShutDownHook(inputStreamCloserHook);
182             }
183         }
184 
185         @Override
186         @SuppressWarnings("checkstyle:innerassignment")
187         public void run() {
188             for (Closeable closeable; (closeable = testProvidingInputStream.poll()) != null; ) {
189                 try {
190                     closeable.close();
191                 } catch (IOException | RuntimeException e) {
192                     // This error does not fail a test and does not necessarily mean that the forked JVM std/out stream
193                     // was not closed, see ThreadedStreamConsumer. This error means that JVM wrote messages to a native
194                     // stream which could not be parsed or report failed. The tests may still correctly run nevertheless
195                     // this exception happened => warning on console. The user would see hint to check dump file only
196                     // if tests failed, but if this does not happen then printing warning to console is the only way to
197                     // inform the users.
198                     String msg = "ForkStarter IOException: " + e.getLocalizedMessage() + ".";
199                     File reportsDir = reportMerger.getReportsDirectory();
200                     File dump =
201                             InPluginProcessDumpSingleton.getSingleton().dumpStreamException(e, msg, reportsDir, jvmRun);
202                     log.warning(msg + " See the dump file " + dump.getAbsolutePath());
203                 }
204             }
205         }
206 
207         @Override
208         public void close() {
209             try {
210                 run();
211             } finally {
212                 testProvidingInputStream.clear();
213                 if (inputStreamCloserHook != null) {
214                     removeShutdownHook(inputStreamCloserHook);
215                 }
216             }
217         }
218 
219         void addCloseable(Closeable closeable) {
220             testProvidingInputStream.add(closeable);
221         }
222     }
223 
224     public ForkStarter(
225             ProviderConfiguration providerConfiguration,
226             StartupConfiguration startupConfiguration,
227             ForkConfiguration forkConfiguration,
228             int forkedProcessTimeoutInSeconds,
229             StartupReportConfiguration startupReportConfiguration,
230             ConsoleLogger log) {
231         this.forkConfiguration = forkConfiguration;
232         this.providerConfiguration = providerConfiguration;
233         this.forkedProcessTimeoutInSeconds = forkedProcessTimeoutInSeconds;
234         this.startupConfiguration = startupConfiguration;
235         this.startupReportConfiguration = startupReportConfiguration;
236         this.log = log;
237         reportMerger = new DefaultReporterFactory(startupReportConfiguration, log);
238         reportMerger.runStarting();
239         defaultReporterFactories = new ConcurrentLinkedQueue<>();
240         currentForkClients = new ConcurrentLinkedQueue<>();
241         timeoutCheckScheduler = createTimeoutCheckScheduler();
242         triggerTimeoutCheck();
243     }
244 
245     public RunResult run(@Nonnull SurefireProperties effectiveSystemProperties, @Nonnull DefaultScanResult scanResult)
246             throws SurefireBooterForkException {
247         try {
248             Map<String, String> providerProperties = providerConfiguration.getProviderProperties();
249             scanResult.writeTo(providerProperties);
250             return isForkOnce() ? run(effectiveSystemProperties, providerProperties) : run(effectiveSystemProperties);
251         } finally {
252             reportMerger.mergeFromOtherFactories(defaultReporterFactories);
253             reportMerger.close();
254             pingThreadScheduler.shutdownNow();
255             timeoutCheckScheduler.shutdownNow();
256             for (String line : logsAtEnd) {
257                 log.warning(line);
258             }
259         }
260     }
261 
262     public void killOrphanForks() {
263         for (ForkClient fork : currentForkClients) {
264             fork.kill();
265         }
266     }
267 
268     private RunResult run(SurefireProperties effectiveSystemProps, Map<String, String> providerProperties)
269             throws SurefireBooterForkException {
270         TestLessInputStreamBuilder builder = new TestLessInputStreamBuilder();
271         TestLessInputStream stream = builder.build();
272         Thread shutdown = createImmediateShutdownHookThread(builder, providerConfiguration.getShutdown());
273         ScheduledFuture<?> ping = triggerPingTimerForShutdown(builder);
274         int forkNumber = drawNumber();
275         PropertiesWrapper props = new PropertiesWrapper(providerProperties);
276         try {
277             addShutDownHook(shutdown);
278             DefaultReporterFactory forkedReporterFactory =
279                     new DefaultReporterFactory(startupReportConfiguration, log, forkNumber);
280             defaultReporterFactories.add(forkedReporterFactory);
281             ForkClient forkClient = new ForkClient(forkedReporterFactory, stream, forkNumber);
282             ForkNodeFactory node = forkConfiguration.getForkNodeFactory();
283             return forkConfiguration.getPluginPlatform().isShutdown()
284                     ? new RunResult(0, 0, 0, 0)
285                     : fork(null, props, forkClient, effectiveSystemProps, forkNumber, stream, node, false);
286         } finally {
287             returnNumber(forkNumber);
288             removeShutdownHook(shutdown);
289             ping.cancel(true);
290             builder.removeStream(stream);
291         }
292     }
293 
294     private RunResult run(SurefireProperties effectiveSystemProperties) throws SurefireBooterForkException {
295         return forkConfiguration.isReuseForks()
296                 ? runSuitesForkOnceMultiple(effectiveSystemProperties, forkConfiguration.getForkCount())
297                 : runSuitesForkPerTestSet(effectiveSystemProperties, forkConfiguration.getForkCount());
298     }
299 
300     private boolean isForkOnce() {
301         return forkConfiguration.isReuseForks() && (forkConfiguration.getForkCount() == 1 || hasSuiteXmlFiles());
302     }
303 
304     private boolean hasSuiteXmlFiles() {
305         TestRequest testSuiteDefinition = providerConfiguration.getTestSuiteDefinition();
306         return testSuiteDefinition != null
307                 && !testSuiteDefinition.getSuiteXmlFiles().isEmpty();
308     }
309 
310     @SuppressWarnings("checkstyle:magicnumber")
311     private RunResult runSuitesForkOnceMultiple(SurefireProperties effectiveSystemProps, int forkCount)
312             throws SurefireBooterForkException {
313         ThreadPoolExecutor executor =
314                 new ThreadPoolExecutor(forkCount, forkCount, 60L, SECONDS, new ArrayBlockingQueue<>(forkCount));
315         executor.setThreadFactory(FORKED_JVM_DAEMON_THREAD_FACTORY);
316 
317         Queue<String> tests = new ConcurrentLinkedQueue<>();
318 
319         for (Class<?> clazz : getSuitesIterator()) {
320             tests.add(clazz.getName());
321         }
322 
323         Queue<TestProvidingInputStream> forks = new ConcurrentLinkedQueue<>();
324 
325         for (int forkNum = 0, total = min(forkCount, tests.size()); forkNum < total; forkNum++) {
326             forks.add(new TestProvidingInputStream(tests));
327         }
328 
329         Thread shutdown = createShutdownHookThread(forks, providerConfiguration.getShutdown());
330         addShutDownHook(shutdown);
331         ScheduledFuture<?> ping = triggerPingTimerForShutdown(forks);
332 
333         try {
334             int failFastCount = providerConfiguration.getSkipAfterFailureCount();
335             AtomicInteger notifyForksToSkipTestsNow = new AtomicInteger(failFastCount);
336             Collection<Future<RunResult>> results = forks.stream()
337                     .filter(fork -> !forkConfiguration.getPluginPlatform().isShutdown())
338                     .map(fork -> (Callable<RunResult>) () -> {
339                         int forkNumber = drawNumber();
340                         DefaultReporterFactory reporter =
341                                 new DefaultReporterFactory(startupReportConfiguration, log, forkNumber);
342                         defaultReporterFactories.add(reporter);
343                         ForkClient client = new ForkClient(reporter, fork, forkNumber);
344                         client.setStopOnNextTestListener(() ->
345                                 runIfZeroCountDown(() -> notifyStreamsToSkipTests(forks), notifyForksToSkipTestsNow));
346                         Map<String, String> providerProperties = providerConfiguration.getProviderProperties();
347                         PropertiesWrapper keyValues = new PropertiesWrapper(providerProperties);
348                         ForkNodeFactory node = forkConfiguration.getForkNodeFactory();
349                         try {
350                             return fork(null, keyValues, client, effectiveSystemProps, forkNumber, fork, node, true);
351                         } finally {
352                             returnNumber(forkNumber);
353                         }
354                     })
355                     .map(executor::submit)
356                     .collect(toList());
357 
358             return awaitResultsDone(results, executor);
359         } finally {
360             removeShutdownHook(shutdown);
361             ping.cancel(true);
362             closeExecutor(executor);
363         }
364     }
365 
366     private static void notifyStreamsToSkipTests(Collection<? extends NotifiableTestStream> notifiableTestStreams) {
367         for (NotifiableTestStream notifiableTestStream : notifiableTestStreams) {
368             notifiableTestStream.skipSinceNextTest();
369         }
370     }
371 
372     @SuppressWarnings("checkstyle:magicnumber")
373     private RunResult runSuitesForkPerTestSet(SurefireProperties effectiveSystemProps, int forkCount)
374             throws SurefireBooterForkException {
375         TestLessInputStreamBuilder builder = new TestLessInputStreamBuilder();
376         Thread shutdown = createCachableShutdownHookThread(builder, providerConfiguration.getShutdown());
377         ThreadPoolExecutor executor =
378                 new ThreadPoolExecutor(forkCount, forkCount, 60, SECONDS, new LinkedBlockingQueue<>());
379         executor.setThreadFactory(FORKED_JVM_DAEMON_THREAD_FACTORY);
380         ScheduledFuture<?> ping = triggerPingTimerForShutdown(builder);
381         try {
382             addShutDownHook(shutdown);
383             int failFastCount = providerConfiguration.getSkipAfterFailureCount();
384             AtomicInteger notifyForksToSkipTestsNow = new AtomicInteger(failFastCount);
385             Collection<Future<RunResult>> results = stream(((Iterable<?>) getSuitesIterator()).spliterator(), false)
386                     .filter(fork -> !forkConfiguration.getPluginPlatform().isShutdown())
387                     .map(testSet -> (Callable<RunResult>) () -> {
388                         int forkNumber = drawNumber();
389                         DefaultReporterFactory forkedReporterFactory =
390                                 new DefaultReporterFactory(startupReportConfiguration, log, forkNumber);
391                         defaultReporterFactories.add(forkedReporterFactory);
392                         TestLessInputStream stream = builder.build();
393                         ForkClient forkClient = new ForkClient(forkedReporterFactory, stream, forkNumber);
394                         NotifiableTestStream notifiable = builder.getCachableCommands();
395                         forkClient.setStopOnNextTestListener(
396                                 () -> runIfZeroCountDown(notifiable::skipSinceNextTest, notifyForksToSkipTestsNow));
397                         Map<String, String> providerProperties = providerConfiguration.getProviderProperties();
398                         PropertiesWrapper keyValues = new PropertiesWrapper(providerProperties);
399                         ForkNodeFactory node = forkConfiguration.getForkNodeFactory();
400                         try {
401                             return fork(
402                                     testSet,
403                                     keyValues,
404                                     forkClient,
405                                     effectiveSystemProps,
406                                     forkNumber,
407                                     stream,
408                                     node,
409                                     false);
410                         } finally {
411                             returnNumber(forkNumber);
412                             builder.removeStream(stream);
413                         }
414                     })
415                     .map(executor::submit)
416                     .collect(toList());
417 
418             return awaitResultsDone(results, executor);
419         } finally {
420             removeShutdownHook(shutdown);
421             ping.cancel(true);
422             closeExecutor(executor);
423         }
424     }
425 
426     private static RunResult awaitResultsDone(Collection<Future<RunResult>> results, ExecutorService executorService)
427             throws SurefireBooterForkException {
428         RunResult globalResult = new RunResult(0, 0, 0, 0);
429         SurefireBooterForkException exception = null;
430         for (Future<RunResult> result : results) {
431             try {
432                 RunResult cur = result.get();
433                 if (cur != null) {
434                     globalResult = globalResult.aggregate(cur);
435                 } else {
436                     throw new SurefireBooterForkException("No results for " + result.toString());
437                 }
438             } catch (InterruptedException e) {
439                 executorService.shutdownNow();
440                 currentThread().interrupt();
441                 throw new SurefireBooterForkException("Interrupted", e);
442             } catch (ExecutionException e) {
443                 Throwable realException = e.getCause();
444                 if (realException == null) {
445                     if (exception == null) {
446                         exception = new SurefireBooterForkException(EXECUTION_EXCEPTION);
447                     }
448                 } else {
449                     String previousError = "";
450                     if (exception != null
451                             && !EXECUTION_EXCEPTION.equals(
452                                     exception.getLocalizedMessage().trim())) {
453                         previousError = exception.getLocalizedMessage() + "\n";
454                     }
455                     String error = previousError + EXECUTION_EXCEPTION + " " + realException.getLocalizedMessage();
456                     exception = new SurefireBooterForkException(error, realException);
457                 }
458             }
459         }
460 
461         if (exception != null) {
462             throw exception;
463         }
464 
465         return globalResult;
466     }
467 
468     @SuppressWarnings("checkstyle:magicnumber")
469     private void closeExecutor(ExecutorService executorService) throws SurefireBooterForkException {
470         executorService.shutdown();
471         try {
472             // Should stop immediately, as we got all the results if we are here
473             executorService.awaitTermination(60 * 60, SECONDS);
474         } catch (InterruptedException e) {
475             currentThread().interrupt();
476             throw new SurefireBooterForkException("Interrupted", e);
477         }
478     }
479 
480     private RunResult fork(
481             Object testSet,
482             PropertiesWrapper providerProperties,
483             ForkClient forkClient,
484             SurefireProperties effectiveSystemProperties,
485             int forkNumber,
486             AbstractCommandReader commandReader,
487             ForkNodeFactory forkNodeFactory,
488             boolean readTestsFromInStream)
489             throws SurefireBooterForkException {
490         CloseableCloser closer = new CloseableCloser(forkNumber, commandReader);
491         final String tempDir;
492         final File surefireProperties;
493         final File systPropsFile;
494         final ForkChannel forkChannel;
495         File dumpLogDir = replaceForkThreadsInPath(startupReportConfiguration.getReportsDirectory(), forkNumber);
496         try {
497             ForkNodeArguments forkNodeArguments = new ForkedNodeArg(
498                     forkConfiguration.isDebug(),
499                     forkNumber,
500                     dumpLogDir,
501                     randomUUID().toString());
502             forkChannel = forkNodeFactory.createForkChannel(forkNodeArguments);
503             closer.addCloseable(forkChannel);
504             tempDir = forkConfiguration.getTempDirectory().getCanonicalPath();
505             BooterSerializer booterSerializer = new BooterSerializer(forkConfiguration);
506             Long pluginPid = forkConfiguration.getPluginPlatform().getPluginPid();
507             log.debug("Determined Maven Process ID " + pluginPid);
508             String connectionString = forkChannel.getForkNodeConnectionString();
509             log.debug("Fork Channel [" + forkNumber + "] connection string '" + connectionString
510                     + "' for the implementation " + forkChannel.getClass());
511             surefireProperties = booterSerializer.serialize(
512                     providerProperties,
513                     providerConfiguration,
514                     startupConfiguration,
515                     testSet,
516                     readTestsFromInStream,
517                     pluginPid,
518                     forkNumber,
519                     connectionString);
520 
521             if (effectiveSystemProperties != null) {
522                 SurefireProperties filteredProperties =
523                         createCopyAndReplaceForkNumPlaceholder(effectiveSystemProperties, forkNumber);
524 
525                 systPropsFile = writePropertiesFile(
526                         filteredProperties,
527                         forkConfiguration.getTempDirectory(),
528                         "surefire_" + SYSTEM_PROPERTIES_FILE_COUNTER.getAndIncrement(),
529                         forkConfiguration.isDebug());
530             } else {
531                 systPropsFile = null;
532             }
533         } catch (IOException e) {
534             throw new SurefireBooterForkException("Error creating properties files for forking", e);
535         }
536 
537         Commandline cli = forkConfiguration.createCommandLine(startupConfiguration, forkNumber, dumpLogDir);
538 
539         cli.createArg().setValue(tempDir);
540         cli.createArg().setValue(DUMP_FILE_PREFIX + forkNumber);
541         cli.createArg().setValue(surefireProperties.getName());
542         if (systPropsFile != null) {
543             cli.createArg().setValue(systPropsFile.getName());
544         }
545 
546         ThreadedStreamConsumer eventConsumer = new ThreadedStreamConsumer(forkClient);
547         closer.addCloseable(eventConsumer);
548 
549         log.debug("Forking command line: " + cli);
550 
551         Integer result = null;
552         RunResult runResult = null;
553         SurefireBooterForkException booterForkException = null;
554         Stoppable err = null;
555         DefaultReporterFactory reporter = forkClient.getDefaultReporterFactory();
556         currentForkClients.add(forkClient);
557         CountdownCloseable countdownCloseable =
558                 new CountdownCloseable(eventConsumer, forkChannel.getCountdownCloseablePermits());
559         try (CommandlineExecutor exec = new CommandlineExecutor(cli, countdownCloseable)) {
560             forkChannel.tryConnectToClient();
561             CommandlineStreams streams = exec.execute();
562             closer.addCloseable(streams);
563 
564             err = bindErrorStream(forkNumber, countdownCloseable, streams, forkClient.getConsoleOutputReceiver());
565 
566             forkChannel.bindCommandReader(commandReader, streams.getStdInChannel());
567 
568             forkChannel.bindEventHandler(eventConsumer, countdownCloseable, streams.getStdOutChannel());
569 
570             log.debug("Fork Channel [" + forkNumber + "] connected to the client.");
571 
572             result = exec.awaitExit();
573 
574             if (forkClient.hadTimeout()) {
575                 runResult = timeout(reporter.getGlobalRunStatistics().getRunResult());
576             } else if (result != SUCCESS) {
577                 booterForkException =
578                         new SurefireBooterForkException("Error occurred in starting fork, check output in log");
579             }
580         } catch (InterruptedException e) {
581             log.error("Closing the streams after (InterruptedException) '" + e.getLocalizedMessage() + "'");
582             // maybe implement it in the Future.cancel() of the extension or similar
583             forkChannel.disable();
584             if (err != null) {
585                 err.disable();
586             }
587         } catch (Exception e) {
588             // CommandLineException from pipes and IOException from sockets
589             runResult = failure(reporter.getGlobalRunStatistics().getRunResult(), e);
590             String cliErr = e.getLocalizedMessage();
591             Throwable cause = e.getCause();
592             booterForkException =
593                     new SurefireBooterForkException("Error while executing forked tests.", cliErr, cause, runResult);
594         } finally {
595             log.debug("Closing the fork " + forkNumber + " after "
596                     + (forkClient.isSaidGoodBye() ? "saying GoodBye." : "not saying Good Bye."));
597             currentForkClients.remove(forkClient);
598             closer.close();
599 
600             if (runResult == null) {
601                 runResult = reporter.getGlobalRunStatistics().getRunResult();
602             }
603             forkClient.close(runResult.isTimeout());
604 
605             if (!runResult.isTimeout()) {
606                 Throwable cause = booterForkException == null ? null : booterForkException.getCause();
607                 String detail = booterForkException == null ? "" : "\n" + booterForkException.getMessage();
608 
609                 if (forkClient.isErrorInFork()) {
610                     StackTraceWriter errorInFork = forkClient.getErrorInFork();
611                     String errorInForkMessage = errorInFork == null
612                             ? null
613                             : errorInFork.getThrowable().getLocalizedMessage();
614                     boolean showStackTrace =
615                             providerConfiguration.getMainCliOptions().contains(SHOW_ERRORS);
616                     String stackTrace = errorInForkMessage;
617                     if (showStackTrace) {
618                         if (errorInFork != null) {
619                             if (stackTrace == null) {
620                                 stackTrace = "";
621                             } else {
622                                 stackTrace += NL;
623                             }
624                             stackTrace += errorInFork.writeTrimmedTraceToString();
625                         }
626                     }
627                     //noinspection ThrowFromFinallyBlock
628                     throw new SurefireBooterForkException(
629                             "There was an error in the forked process"
630                                     + detail
631                                     + (stackTrace == null ? "" : "\n" + stackTrace),
632                             cause);
633                 }
634                 if (!forkClient.isSaidGoodBye()) {
635                     String errorCode = result == null ? "" : "\nProcess Exit Code: " + result;
636                     String testsInProgress = forkClient.hasTestsInProgress() ? "\nCrashed tests:" : "";
637                     for (String test : forkClient.testsInProgress()) {
638                         testsInProgress += "\n" + test;
639                     }
640                     // noinspection ThrowFromFinallyBlock
641                     throw new SurefireBooterForkException(
642                             "The forked VM terminated without properly saying goodbye. VM crash or System.exit called?"
643                                     + "\nCommand was " + cli.toString() + detail + errorCode + testsInProgress,
644                             cause);
645                 }
646             }
647 
648             if (booterForkException != null) {
649                 throw booterForkException;
650             }
651         }
652 
653         return runResult;
654     }
655 
656     private static Stoppable bindErrorStream(
657             int forkNumber, CountdownCloseable countdownCloseable, CommandlineStreams streams, Object errorStreamLock) {
658         EventHandler<String> errConsumer = new NativeStdErrStreamConsumer(errorStreamLock);
659         LineConsumerThread stdErr = new LineConsumerThread(
660                 "fork-" + forkNumber + "-err-thread", streams.getStdErrChannel(), errConsumer, countdownCloseable);
661         stdErr.start();
662         return stdErr;
663     }
664 
665     private Iterable<Class<?>> getSuitesIterator() throws SurefireBooterForkException {
666         try {
667             AbstractPathConfiguration classpathConfiguration = startupConfiguration.getClasspathConfiguration();
668             ClassLoader unifiedClassLoader = classpathConfiguration.createMergedClassLoader();
669 
670             CommonReflector commonReflector = new CommonReflector(unifiedClassLoader);
671             Object reporterFactory = commonReflector.createReportingReporterFactory(startupReportConfiguration, log);
672 
673             ProviderFactory providerFactory = new ProviderFactory(
674                     startupConfiguration, providerConfiguration, unifiedClassLoader, reporterFactory);
675             SurefireProvider surefireProvider = providerFactory.createProvider(false);
676             return surefireProvider.getSuites();
677         } catch (SurefireExecutionException e) {
678             throw new SurefireBooterForkException("Unable to create classloader to find test suites", e);
679         }
680     }
681 
682     private static Thread createImmediateShutdownHookThread(
683             final TestLessInputStreamBuilder builder, final Shutdown shutdownType) {
684         return SHUTDOWN_HOOK_THREAD_FACTORY.newThread(new Runnable() {
685             @Override
686             public void run() {
687                 builder.getImmediateCommands().shutdown(shutdownType);
688             }
689         });
690     }
691 
692     private static Thread createCachableShutdownHookThread(
693             final TestLessInputStreamBuilder builder, final Shutdown shutdownType) {
694         return SHUTDOWN_HOOK_THREAD_FACTORY.newThread(new Runnable() {
695             @Override
696             public void run() {
697                 builder.getCachableCommands().shutdown(shutdownType);
698             }
699         });
700     }
701 
702     private static Thread createShutdownHookThread(
703             final Iterable<TestProvidingInputStream> streams, final Shutdown shutdownType) {
704         return SHUTDOWN_HOOK_THREAD_FACTORY.newThread(new Runnable() {
705             @Override
706             public void run() {
707                 for (TestProvidingInputStream stream : streams) {
708                     stream.shutdown(shutdownType);
709                 }
710             }
711         });
712     }
713 
714     private static ScheduledExecutorService createPingScheduler() {
715         ThreadFactory threadFactory = newDaemonThreadFactory("ping-timer-" + PING_IN_SECONDS + "s");
716         return newScheduledThreadPool(1, threadFactory);
717     }
718 
719     private static ScheduledExecutorService createTimeoutCheckScheduler() {
720         ThreadFactory threadFactory = newDaemonThreadFactory("timeout-check-timer");
721         return newScheduledThreadPool(1, threadFactory);
722     }
723 
724     private ScheduledFuture<?> triggerPingTimerForShutdown(final TestLessInputStreamBuilder builder) {
725         return pingThreadScheduler.scheduleWithFixedDelay(
726                 new Runnable() {
727                     @Override
728                     public void run() {
729                         builder.getImmediateCommands().noop();
730                     }
731                 },
732                 0,
733                 PING_IN_SECONDS,
734                 SECONDS);
735     }
736 
737     private ScheduledFuture<?> triggerPingTimerForShutdown(final Iterable<TestProvidingInputStream> streams) {
738         return pingThreadScheduler.scheduleWithFixedDelay(
739                 new Runnable() {
740                     @Override
741                     public void run() {
742                         for (TestProvidingInputStream stream : streams) {
743                             stream.noop();
744                         }
745                     }
746                 },
747                 0,
748                 PING_IN_SECONDS,
749                 SECONDS);
750     }
751 
752     private ScheduledFuture<?> triggerTimeoutCheck() {
753         return timeoutCheckScheduler.scheduleWithFixedDelay(
754                 new Runnable() {
755                     @Override
756                     public void run() {
757                         long systemTime = currentTimeMillis();
758                         for (ForkClient forkClient : currentForkClients) {
759                             forkClient.tryToTimeout(systemTime, forkedProcessTimeoutInSeconds);
760                         }
761                     }
762                 },
763                 0,
764                 TIMEOUT_CHECK_PERIOD_MILLIS,
765                 MILLISECONDS);
766     }
767 
768     private final class ForkedNodeArg implements ForkNodeArguments {
769         private final int forkChannelId;
770         private final File dumpLogDir;
771         private final String sessionId;
772         private final boolean debug;
773 
774         ForkedNodeArg(boolean debug, int forkChannelId, File dumpLogDir, String sessionId) {
775             this.debug = debug;
776             this.forkChannelId = forkChannelId;
777             this.dumpLogDir = dumpLogDir;
778             this.sessionId = sessionId;
779         }
780 
781         @Nonnull
782         @Override
783         public String getSessionId() {
784             return sessionId;
785         }
786 
787         @Override
788         public int getForkChannelId() {
789             return forkChannelId;
790         }
791 
792         @Override
793         @Nonnull
794         public File dumpStreamText(@Nonnull String text) {
795             return InPluginProcessDumpSingleton.getSingleton().dumpStreamText(text, dumpLogDir, forkChannelId);
796         }
797 
798         @Nonnull
799         @Override
800         public File dumpStreamException(@Nonnull Throwable t) {
801             return InPluginProcessDumpSingleton.getSingleton()
802                     .dumpStreamException(t, t.getLocalizedMessage(), dumpLogDir, forkChannelId);
803         }
804 
805         @Override
806         public void logWarningAtEnd(@Nonnull String text) {
807             logsAtEnd.add(text);
808         }
809 
810         @Nonnull
811         @Override
812         public ConsoleLogger getConsoleLogger() {
813             return log;
814         }
815 
816         @Nonnull
817         @Override
818         public Object getConsoleLock() {
819             return log;
820         }
821 
822         @Override
823         public File getEventStreamBinaryFile() {
824             return debug
825                     ? InPluginProcessDumpSingleton.getSingleton().getEventStreamBinaryFile(dumpLogDir, forkChannelId)
826                     : null;
827         }
828 
829         @Override
830         public File getCommandStreamBinaryFile() {
831             return null;
832         }
833     }
834 }