View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.booterclient;
20  
21  import javax.annotation.Nonnull;
22  
23  import java.io.Closeable;
24  import java.io.File;
25  import java.io.IOException;
26  import java.util.Collection;
27  import java.util.Map;
28  import java.util.Queue;
29  import java.util.Set;
30  import java.util.concurrent.ArrayBlockingQueue;
31  import java.util.concurrent.Callable;
32  import java.util.concurrent.ConcurrentLinkedQueue;
33  import java.util.concurrent.ConcurrentSkipListSet;
34  import java.util.concurrent.ExecutionException;
35  import java.util.concurrent.ExecutorService;
36  import java.util.concurrent.Future;
37  import java.util.concurrent.LinkedBlockingQueue;
38  import java.util.concurrent.ScheduledExecutorService;
39  import java.util.concurrent.ScheduledFuture;
40  import java.util.concurrent.ThreadFactory;
41  import java.util.concurrent.ThreadPoolExecutor;
42  import java.util.concurrent.atomic.AtomicInteger;
43  
44  import org.apache.maven.plugin.surefire.CommonReflector;
45  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
46  import org.apache.maven.plugin.surefire.SurefireProperties;
47  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.AbstractCommandReader;
48  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.Commandline;
49  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.NotifiableTestStream;
50  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.TestLessInputStream;
51  import org.apache.maven.plugin.surefire.booterclient.lazytestprovider.TestProvidingInputStream;
52  import org.apache.maven.plugin.surefire.booterclient.output.ForkClient;
53  import org.apache.maven.plugin.surefire.booterclient.output.InPluginProcessDumpSingleton;
54  import org.apache.maven.plugin.surefire.booterclient.output.NativeStdErrStreamConsumer;
55  import org.apache.maven.plugin.surefire.booterclient.output.ThreadedStreamConsumer;
56  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
57  import org.apache.maven.plugin.surefire.report.DefaultReporterFactory;
58  import org.apache.maven.plugin.surefire.report.ReportsMerger;
59  import org.apache.maven.surefire.api.booter.Shutdown;
60  import org.apache.maven.surefire.api.fork.ForkNodeArguments;
61  import org.apache.maven.surefire.api.provider.SurefireProvider;
62  import org.apache.maven.surefire.api.report.StackTraceWriter;
63  import org.apache.maven.surefire.api.suite.RunResult;
64  import org.apache.maven.surefire.api.util.DefaultScanResult;
65  import org.apache.maven.surefire.booter.AbstractPathConfiguration;
66  import org.apache.maven.surefire.booter.PropertiesWrapper;
67  import org.apache.maven.surefire.booter.ProviderConfiguration;
68  import org.apache.maven.surefire.booter.ProviderFactory;
69  import org.apache.maven.surefire.booter.StartupConfiguration;
70  import org.apache.maven.surefire.booter.SurefireBooterForkException;
71  import org.apache.maven.surefire.booter.SurefireExecutionException;
72  import org.apache.maven.surefire.extensions.EventHandler;
73  import org.apache.maven.surefire.extensions.ForkChannel;
74  import org.apache.maven.surefire.extensions.ForkNodeFactory;
75  import org.apache.maven.surefire.extensions.Stoppable;
76  import org.apache.maven.surefire.extensions.util.CommandlineExecutor;
77  import org.apache.maven.surefire.extensions.util.CommandlineStreams;
78  import org.apache.maven.surefire.extensions.util.CountdownCloseable;
79  import org.apache.maven.surefire.extensions.util.LineConsumerThread;
80  
81  import static java.lang.StrictMath.min;
82  import static java.lang.System.currentTimeMillis;
83  import static java.lang.Thread.currentThread;
84  import static java.util.Collections.addAll;
85  import static java.util.UUID.randomUUID;
86  import static java.util.concurrent.Executors.newScheduledThreadPool;
87  import static java.util.concurrent.TimeUnit.MILLISECONDS;
88  import static java.util.concurrent.TimeUnit.SECONDS;
89  import static java.util.stream.Collectors.toList;
90  import static java.util.stream.StreamSupport.stream;
91  import static org.apache.maven.plugin.surefire.AbstractSurefireMojo.createCopyAndReplaceForkNumPlaceholder;
92  import static org.apache.maven.plugin.surefire.SurefireHelper.DUMP_FILE_PREFIX;
93  import static org.apache.maven.plugin.surefire.SurefireHelper.replaceForkThreadsInPath;
94  import static org.apache.maven.plugin.surefire.booterclient.ForkNumberBucket.drawNumber;
95  import static org.apache.maven.plugin.surefire.booterclient.ForkNumberBucket.returnNumber;
96  import static org.apache.maven.plugin.surefire.booterclient.lazytestprovider.TestLessInputStream.TestLessInputStreamBuilder;
97  import static org.apache.maven.surefire.api.cli.CommandLineOption.SHOW_ERRORS;
98  import static org.apache.maven.surefire.api.suite.RunResult.SUCCESS;
99  import static org.apache.maven.surefire.api.suite.RunResult.failure;
100 import static org.apache.maven.surefire.api.suite.RunResult.timeout;
101 import static org.apache.maven.surefire.api.util.internal.ConcurrencyUtils.runIfZeroCountDown;
102 import static org.apache.maven.surefire.api.util.internal.DaemonThreadFactory.newDaemonThread;
103 import static org.apache.maven.surefire.api.util.internal.DaemonThreadFactory.newDaemonThreadFactory;
104 import static org.apache.maven.surefire.api.util.internal.StringUtils.NL;
105 import static org.apache.maven.surefire.booter.SystemPropertyManager.writePropertiesFile;
106 import static org.apache.maven.surefire.shared.utils.cli.ShutdownHookUtils.addShutDownHook;
107 import static org.apache.maven.surefire.shared.utils.cli.ShutdownHookUtils.removeShutdownHook;
108 
109 /**
110  * Starts the fork or runs in-process.
111  * <br>
112  * Lives only on the plugin-side (not present in remote vms)
113  * <br>
114  * Knows how to fork new vms and also how to delegate non-forking invocation to SurefireStarter directly
115  *
116  * @author Jason van Zyl
117  * @author Emmanuel Venisse
118  * @author Brett Porter
119  * @author Dan Fabulich
120  * @author Carlos Sanchez
121  * @author Kristian Rosenvold
122  */
123 public class ForkStarter {
124     private static final String EXECUTION_EXCEPTION = "ExecutionException";
125 
126     private static final long PING_IN_SECONDS = 10;
127 
128     private static final int TIMEOUT_CHECK_PERIOD_MILLIS = 100;
129 
130     private static final ThreadFactory FORKED_JVM_DAEMON_THREAD_FACTORY =
131             newDaemonThreadFactory("surefire-fork-starter");
132 
133     private static final ThreadFactory SHUTDOWN_HOOK_THREAD_FACTORY =
134             newDaemonThreadFactory("surefire-jvm-killer-shutdownhook");
135 
136     private static final AtomicInteger SYSTEM_PROPERTIES_FILE_COUNTER = new AtomicInteger();
137 
138     private final Set<String> logsAtEnd = new ConcurrentSkipListSet<>();
139 
140     private final ScheduledExecutorService pingThreadScheduler = createPingScheduler();
141 
142     private final ScheduledExecutorService timeoutCheckScheduler;
143 
144     private final Queue<ForkClient> currentForkClients;
145 
146     private final int forkedProcessTimeoutInSeconds;
147 
148     private final ProviderConfiguration providerConfiguration;
149 
150     private final StartupConfiguration startupConfiguration;
151 
152     private final ForkConfiguration forkConfiguration;
153 
154     private final StartupReportConfiguration startupReportConfiguration;
155 
156     private final ConsoleLogger log;
157 
158     private final ReportsMerger reportMerger;
159 
160     private final Collection<DefaultReporterFactory> defaultReporterFactories;
161 
162     /**
163      * Closes stuff, with a shutdown hook to make sure things really get closed.
164      */
165     private final class CloseableCloser implements Runnable, Closeable {
166         private final int jvmRun;
167 
168         private final Queue<Closeable> testProvidingInputStream;
169 
170         private final Thread inputStreamCloserHook;
171 
172         CloseableCloser(int jvmRun, Closeable... testProvidingInputStream) {
173             this.jvmRun = jvmRun;
174             this.testProvidingInputStream = new ConcurrentLinkedQueue<>();
175             addAll(this.testProvidingInputStream, testProvidingInputStream);
176             if (this.testProvidingInputStream.isEmpty()) {
177                 inputStreamCloserHook = null;
178             } else {
179                 inputStreamCloserHook = newDaemonThread(this, "closer-shutdown-hook");
180                 addShutDownHook(inputStreamCloserHook);
181             }
182         }
183 
184         @Override
185         @SuppressWarnings("checkstyle:innerassignment")
186         public void run() {
187             for (Closeable closeable; (closeable = testProvidingInputStream.poll()) != null; ) {
188                 try {
189                     closeable.close();
190                 } catch (IOException | RuntimeException e) {
191                     // This error does not fail a test and does not necessarily mean that the forked JVM std/out stream
192                     // was not closed, see ThreadedStreamConsumer. This error means that JVM wrote messages to a native
193                     // stream which could not be parsed or report failed. The tests may still correctly run nevertheless
194                     // this exception happened => warning on console. The user would see hint to check dump file only
195                     // if tests failed, but if this does not happen then printing warning to console is the only way to
196                     // inform the users.
197                     String msg = "ForkStarter IOException: " + e.getLocalizedMessage() + ".";
198                     File reportsDir = reportMerger.getReportsDirectory();
199                     File dump =
200                             InPluginProcessDumpSingleton.getSingleton().dumpStreamException(e, msg, reportsDir, jvmRun);
201                     log.warning(msg + " See the dump file " + dump.getAbsolutePath());
202                 }
203             }
204         }
205 
206         @Override
207         public void close() {
208             try {
209                 run();
210             } finally {
211                 testProvidingInputStream.clear();
212                 if (inputStreamCloserHook != null) {
213                     removeShutdownHook(inputStreamCloserHook);
214                 }
215             }
216         }
217 
218         void addCloseable(Closeable closeable) {
219             testProvidingInputStream.add(closeable);
220         }
221     }
222 
223     public ForkStarter(
224             ProviderConfiguration providerConfiguration,
225             StartupConfiguration startupConfiguration,
226             ForkConfiguration forkConfiguration,
227             int forkedProcessTimeoutInSeconds,
228             StartupReportConfiguration startupReportConfiguration,
229             ConsoleLogger log) {
230         this.forkConfiguration = forkConfiguration;
231         this.providerConfiguration = providerConfiguration;
232         this.forkedProcessTimeoutInSeconds = forkedProcessTimeoutInSeconds;
233         this.startupConfiguration = startupConfiguration;
234         this.startupReportConfiguration = startupReportConfiguration;
235         this.log = log;
236         reportMerger = new DefaultReporterFactory(startupReportConfiguration, log);
237         reportMerger.runStarting();
238         defaultReporterFactories = new ConcurrentLinkedQueue<>();
239         currentForkClients = new ConcurrentLinkedQueue<>();
240         timeoutCheckScheduler = createTimeoutCheckScheduler();
241         triggerTimeoutCheck();
242     }
243 
244     public RunResult run(@Nonnull SurefireProperties effectiveSystemProperties, @Nonnull DefaultScanResult scanResult)
245             throws SurefireBooterForkException {
246         try {
247             Map<String, String> providerProperties = providerConfiguration.getProviderProperties();
248             scanResult.writeTo(providerProperties);
249             return isForkOnce() ? run(effectiveSystemProperties, providerProperties) : run(effectiveSystemProperties);
250         } finally {
251             reportMerger.mergeFromOtherFactories(defaultReporterFactories);
252             reportMerger.close();
253             pingThreadScheduler.shutdownNow();
254             timeoutCheckScheduler.shutdownNow();
255             for (String line : logsAtEnd) {
256                 log.warning(line);
257             }
258         }
259     }
260 
261     public void killOrphanForks() {
262         for (ForkClient fork : currentForkClients) {
263             fork.kill();
264         }
265     }
266 
267     private RunResult run(SurefireProperties effectiveSystemProps, Map<String, String> providerProperties)
268             throws SurefireBooterForkException {
269         TestLessInputStreamBuilder builder = new TestLessInputStreamBuilder();
270         TestLessInputStream stream = builder.build();
271         Thread shutdown = createImmediateShutdownHookThread(builder, providerConfiguration.getShutdown());
272         ScheduledFuture<?> ping = triggerPingTimerForShutdown(builder);
273         int forkNumber = drawNumber();
274         PropertiesWrapper props = new PropertiesWrapper(providerProperties);
275         try {
276             addShutDownHook(shutdown);
277             DefaultReporterFactory forkedReporterFactory =
278                     new DefaultReporterFactory(startupReportConfiguration, log, forkNumber);
279             defaultReporterFactories.add(forkedReporterFactory);
280             ForkClient forkClient = new ForkClient(forkedReporterFactory, stream, forkNumber);
281             ForkNodeFactory node = forkConfiguration.getForkNodeFactory();
282             return forkConfiguration.getPluginPlatform().isShutdown()
283                     ? new RunResult(0, 0, 0, 0)
284                     : fork(null, props, forkClient, effectiveSystemProps, forkNumber, stream, node, false);
285         } finally {
286             returnNumber(forkNumber);
287             removeShutdownHook(shutdown);
288             ping.cancel(true);
289             builder.removeStream(stream);
290         }
291     }
292 
293     private RunResult run(SurefireProperties effectiveSystemProperties) throws SurefireBooterForkException {
294         return forkConfiguration.isReuseForks()
295                 ? runSuitesForkOnceMultiple(effectiveSystemProperties, forkConfiguration.getForkCount())
296                 : runSuitesForkPerTestSet(effectiveSystemProperties, forkConfiguration.getForkCount());
297     }
298 
299     private boolean isForkOnce() {
300         return forkConfiguration.isReuseForks() && forkConfiguration.getForkCount() == 1;
301     }
302 
303     @SuppressWarnings("checkstyle:magicnumber")
304     private RunResult runSuitesForkOnceMultiple(SurefireProperties effectiveSystemProps, int forkCount)
305             throws SurefireBooterForkException {
306         ThreadPoolExecutor executor =
307                 new ThreadPoolExecutor(forkCount, forkCount, 60L, SECONDS, new ArrayBlockingQueue<>(forkCount));
308         executor.setThreadFactory(FORKED_JVM_DAEMON_THREAD_FACTORY);
309 
310         Queue<String> tests = new ConcurrentLinkedQueue<>();
311 
312         for (Class<?> clazz : getSuitesIterator()) {
313             tests.add(clazz.getName());
314         }
315 
316         Queue<TestProvidingInputStream> forks = new ConcurrentLinkedQueue<>();
317 
318         for (int forkNum = 0, total = min(forkCount, tests.size()); forkNum < total; forkNum++) {
319             forks.add(new TestProvidingInputStream(tests));
320         }
321 
322         Thread shutdown = createShutdownHookThread(forks, providerConfiguration.getShutdown());
323         addShutDownHook(shutdown);
324         ScheduledFuture<?> ping = triggerPingTimerForShutdown(forks);
325 
326         try {
327             int failFastCount = providerConfiguration.getSkipAfterFailureCount();
328             AtomicInteger notifyForksToSkipTestsNow = new AtomicInteger(failFastCount);
329             Collection<Future<RunResult>> results = forks.stream()
330                     .filter(fork -> !forkConfiguration.getPluginPlatform().isShutdown())
331                     .map(fork -> (Callable<RunResult>) () -> {
332                         int forkNumber = drawNumber();
333                         DefaultReporterFactory reporter =
334                                 new DefaultReporterFactory(startupReportConfiguration, log, forkNumber);
335                         defaultReporterFactories.add(reporter);
336                         ForkClient client = new ForkClient(reporter, fork, forkNumber);
337                         client.setStopOnNextTestListener(() ->
338                                 runIfZeroCountDown(() -> notifyStreamsToSkipTests(forks), notifyForksToSkipTestsNow));
339                         Map<String, String> providerProperties = providerConfiguration.getProviderProperties();
340                         PropertiesWrapper keyValues = new PropertiesWrapper(providerProperties);
341                         ForkNodeFactory node = forkConfiguration.getForkNodeFactory();
342                         try {
343                             return fork(null, keyValues, client, effectiveSystemProps, forkNumber, fork, node, true);
344                         } finally {
345                             returnNumber(forkNumber);
346                         }
347                     })
348                     .map(executor::submit)
349                     .collect(toList());
350 
351             return awaitResultsDone(results, executor);
352         } finally {
353             removeShutdownHook(shutdown);
354             ping.cancel(true);
355             closeExecutor(executor);
356         }
357     }
358 
359     private static void notifyStreamsToSkipTests(Collection<? extends NotifiableTestStream> notifiableTestStreams) {
360         for (NotifiableTestStream notifiableTestStream : notifiableTestStreams) {
361             notifiableTestStream.skipSinceNextTest();
362         }
363     }
364 
365     @SuppressWarnings("checkstyle:magicnumber")
366     private RunResult runSuitesForkPerTestSet(SurefireProperties effectiveSystemProps, int forkCount)
367             throws SurefireBooterForkException {
368         TestLessInputStreamBuilder builder = new TestLessInputStreamBuilder();
369         Thread shutdown = createCachableShutdownHookThread(builder, providerConfiguration.getShutdown());
370         ThreadPoolExecutor executor =
371                 new ThreadPoolExecutor(forkCount, forkCount, 60, SECONDS, new LinkedBlockingQueue<>());
372         executor.setThreadFactory(FORKED_JVM_DAEMON_THREAD_FACTORY);
373         ScheduledFuture<?> ping = triggerPingTimerForShutdown(builder);
374         try {
375             addShutDownHook(shutdown);
376             int failFastCount = providerConfiguration.getSkipAfterFailureCount();
377             AtomicInteger notifyForksToSkipTestsNow = new AtomicInteger(failFastCount);
378             Collection<Future<RunResult>> results = stream(((Iterable<?>) getSuitesIterator()).spliterator(), false)
379                     .filter(fork -> !forkConfiguration.getPluginPlatform().isShutdown())
380                     .map(testSet -> (Callable<RunResult>) () -> {
381                         int forkNumber = drawNumber();
382                         DefaultReporterFactory forkedReporterFactory =
383                                 new DefaultReporterFactory(startupReportConfiguration, log, forkNumber);
384                         defaultReporterFactories.add(forkedReporterFactory);
385                         TestLessInputStream stream = builder.build();
386                         ForkClient forkClient = new ForkClient(forkedReporterFactory, stream, forkNumber);
387                         NotifiableTestStream notifiable = builder.getCachableCommands();
388                         forkClient.setStopOnNextTestListener(
389                                 () -> runIfZeroCountDown(notifiable::skipSinceNextTest, notifyForksToSkipTestsNow));
390                         Map<String, String> providerProperties = providerConfiguration.getProviderProperties();
391                         PropertiesWrapper keyValues = new PropertiesWrapper(providerProperties);
392                         ForkNodeFactory node = forkConfiguration.getForkNodeFactory();
393                         try {
394                             return fork(
395                                     testSet,
396                                     keyValues,
397                                     forkClient,
398                                     effectiveSystemProps,
399                                     forkNumber,
400                                     stream,
401                                     node,
402                                     false);
403                         } finally {
404                             returnNumber(forkNumber);
405                             builder.removeStream(stream);
406                         }
407                     })
408                     .map(executor::submit)
409                     .collect(toList());
410 
411             return awaitResultsDone(results, executor);
412         } finally {
413             removeShutdownHook(shutdown);
414             ping.cancel(true);
415             closeExecutor(executor);
416         }
417     }
418 
419     private static RunResult awaitResultsDone(Collection<Future<RunResult>> results, ExecutorService executorService)
420             throws SurefireBooterForkException {
421         RunResult globalResult = new RunResult(0, 0, 0, 0);
422         SurefireBooterForkException exception = null;
423         for (Future<RunResult> result : results) {
424             try {
425                 RunResult cur = result.get();
426                 if (cur != null) {
427                     globalResult = globalResult.aggregate(cur);
428                 } else {
429                     throw new SurefireBooterForkException("No results for " + result.toString());
430                 }
431             } catch (InterruptedException e) {
432                 executorService.shutdownNow();
433                 currentThread().interrupt();
434                 throw new SurefireBooterForkException("Interrupted", e);
435             } catch (ExecutionException e) {
436                 Throwable realException = e.getCause();
437                 if (realException == null) {
438                     if (exception == null) {
439                         exception = new SurefireBooterForkException(EXECUTION_EXCEPTION);
440                     }
441                 } else {
442                     String previousError = "";
443                     if (exception != null
444                             && !EXECUTION_EXCEPTION.equals(
445                                     exception.getLocalizedMessage().trim())) {
446                         previousError = exception.getLocalizedMessage() + "\n";
447                     }
448                     String error = previousError + EXECUTION_EXCEPTION + " " + realException.getLocalizedMessage();
449                     exception = new SurefireBooterForkException(error, realException);
450                 }
451             }
452         }
453 
454         if (exception != null) {
455             throw exception;
456         }
457 
458         return globalResult;
459     }
460 
461     @SuppressWarnings("checkstyle:magicnumber")
462     private void closeExecutor(ExecutorService executorService) throws SurefireBooterForkException {
463         executorService.shutdown();
464         try {
465             // Should stop immediately, as we got all the results if we are here
466             executorService.awaitTermination(60 * 60, SECONDS);
467         } catch (InterruptedException e) {
468             currentThread().interrupt();
469             throw new SurefireBooterForkException("Interrupted", e);
470         }
471     }
472 
473     private RunResult fork(
474             Object testSet,
475             PropertiesWrapper providerProperties,
476             ForkClient forkClient,
477             SurefireProperties effectiveSystemProperties,
478             int forkNumber,
479             AbstractCommandReader commandReader,
480             ForkNodeFactory forkNodeFactory,
481             boolean readTestsFromInStream)
482             throws SurefireBooterForkException {
483         CloseableCloser closer = new CloseableCloser(forkNumber, commandReader);
484         final String tempDir;
485         final File surefireProperties;
486         final File systPropsFile;
487         final ForkChannel forkChannel;
488         File dumpLogDir = replaceForkThreadsInPath(startupReportConfiguration.getReportsDirectory(), forkNumber);
489         try {
490             ForkNodeArguments forkNodeArguments = new ForkedNodeArg(
491                     forkConfiguration.isDebug(),
492                     forkNumber,
493                     dumpLogDir,
494                     randomUUID().toString());
495             forkChannel = forkNodeFactory.createForkChannel(forkNodeArguments);
496             closer.addCloseable(forkChannel);
497             tempDir = forkConfiguration.getTempDirectory().getCanonicalPath();
498             BooterSerializer booterSerializer = new BooterSerializer(forkConfiguration);
499             Long pluginPid = forkConfiguration.getPluginPlatform().getPluginPid();
500             log.debug("Determined Maven Process ID " + pluginPid);
501             String connectionString = forkChannel.getForkNodeConnectionString();
502             log.debug("Fork Channel [" + forkNumber + "] connection string '" + connectionString
503                     + "' for the implementation " + forkChannel.getClass());
504             surefireProperties = booterSerializer.serialize(
505                     providerProperties,
506                     providerConfiguration,
507                     startupConfiguration,
508                     testSet,
509                     readTestsFromInStream,
510                     pluginPid,
511                     forkNumber,
512                     connectionString);
513 
514             if (effectiveSystemProperties != null) {
515                 SurefireProperties filteredProperties =
516                         createCopyAndReplaceForkNumPlaceholder(effectiveSystemProperties, forkNumber);
517 
518                 systPropsFile = writePropertiesFile(
519                         filteredProperties,
520                         forkConfiguration.getTempDirectory(),
521                         "surefire_" + SYSTEM_PROPERTIES_FILE_COUNTER.getAndIncrement(),
522                         forkConfiguration.isDebug());
523             } else {
524                 systPropsFile = null;
525             }
526         } catch (IOException e) {
527             throw new SurefireBooterForkException("Error creating properties files for forking", e);
528         }
529 
530         Commandline cli = forkConfiguration.createCommandLine(startupConfiguration, forkNumber, dumpLogDir);
531 
532         cli.createArg().setValue(tempDir);
533         cli.createArg().setValue(DUMP_FILE_PREFIX + forkNumber);
534         cli.createArg().setValue(surefireProperties.getName());
535         if (systPropsFile != null) {
536             cli.createArg().setValue(systPropsFile.getName());
537         }
538 
539         ThreadedStreamConsumer eventConsumer = new ThreadedStreamConsumer(forkClient);
540         closer.addCloseable(eventConsumer);
541 
542         log.debug("Forking command line: " + cli);
543 
544         Integer result = null;
545         RunResult runResult = null;
546         SurefireBooterForkException booterForkException = null;
547         Stoppable err = null;
548         DefaultReporterFactory reporter = forkClient.getDefaultReporterFactory();
549         currentForkClients.add(forkClient);
550         CountdownCloseable countdownCloseable =
551                 new CountdownCloseable(eventConsumer, forkChannel.getCountdownCloseablePermits());
552         try (CommandlineExecutor exec = new CommandlineExecutor(cli, countdownCloseable)) {
553             forkChannel.tryConnectToClient();
554             CommandlineStreams streams = exec.execute();
555             closer.addCloseable(streams);
556 
557             err = bindErrorStream(forkNumber, countdownCloseable, streams, forkClient.getConsoleOutputReceiver());
558 
559             forkChannel.bindCommandReader(commandReader, streams.getStdInChannel());
560 
561             forkChannel.bindEventHandler(eventConsumer, countdownCloseable, streams.getStdOutChannel());
562 
563             log.debug("Fork Channel [" + forkNumber + "] connected to the client.");
564 
565             result = exec.awaitExit();
566 
567             if (forkClient.hadTimeout()) {
568                 runResult = timeout(reporter.getGlobalRunStatistics().getRunResult());
569             } else if (result != SUCCESS) {
570                 booterForkException =
571                         new SurefireBooterForkException("Error occurred in starting fork, check output in log");
572             }
573         } catch (InterruptedException e) {
574             log.error("Closing the streams after (InterruptedException) '" + e.getLocalizedMessage() + "'");
575             // maybe implement it in the Future.cancel() of the extension or similar
576             forkChannel.disable();
577             if (err != null) {
578                 err.disable();
579             }
580         } catch (Exception e) {
581             // CommandLineException from pipes and IOException from sockets
582             runResult = failure(reporter.getGlobalRunStatistics().getRunResult(), e);
583             String cliErr = e.getLocalizedMessage();
584             Throwable cause = e.getCause();
585             booterForkException =
586                     new SurefireBooterForkException("Error while executing forked tests.", cliErr, cause, runResult);
587         } finally {
588             log.debug("Closing the fork " + forkNumber + " after "
589                     + (forkClient.isSaidGoodBye() ? "saying GoodBye." : "not saying Good Bye."));
590             currentForkClients.remove(forkClient);
591             closer.close();
592 
593             if (runResult == null) {
594                 runResult = reporter.getGlobalRunStatistics().getRunResult();
595             }
596             forkClient.close(runResult.isTimeout());
597 
598             if (!runResult.isTimeout()) {
599                 Throwable cause = booterForkException == null ? null : booterForkException.getCause();
600                 String detail = booterForkException == null ? "" : "\n" + booterForkException.getMessage();
601 
602                 if (forkClient.isErrorInFork()) {
603                     StackTraceWriter errorInFork = forkClient.getErrorInFork();
604                     String errorInForkMessage = errorInFork == null
605                             ? null
606                             : errorInFork.getThrowable().getLocalizedMessage();
607                     boolean showStackTrace =
608                             providerConfiguration.getMainCliOptions().contains(SHOW_ERRORS);
609                     String stackTrace = errorInForkMessage;
610                     if (showStackTrace) {
611                         if (errorInFork != null) {
612                             if (stackTrace == null) {
613                                 stackTrace = "";
614                             } else {
615                                 stackTrace += NL;
616                             }
617                             stackTrace += errorInFork.writeTrimmedTraceToString();
618                         }
619                     }
620                     //noinspection ThrowFromFinallyBlock
621                     throw new SurefireBooterForkException(
622                             "There was an error in the forked process"
623                                     + detail
624                                     + (stackTrace == null ? "" : "\n" + stackTrace),
625                             cause);
626                 }
627                 if (!forkClient.isSaidGoodBye()) {
628                     String errorCode = result == null ? "" : "\nProcess Exit Code: " + result;
629                     String testsInProgress = forkClient.hasTestsInProgress() ? "\nCrashed tests:" : "";
630                     for (String test : forkClient.testsInProgress()) {
631                         testsInProgress += "\n" + test;
632                     }
633                     // noinspection ThrowFromFinallyBlock
634                     throw new SurefireBooterForkException(
635                             "The forked VM terminated without properly saying goodbye. VM crash or System.exit called?"
636                                     + "\nCommand was " + cli.toString() + detail + errorCode + testsInProgress,
637                             cause);
638                 }
639             }
640 
641             if (booterForkException != null) {
642                 throw booterForkException;
643             }
644         }
645 
646         return runResult;
647     }
648 
649     private static Stoppable bindErrorStream(
650             int forkNumber, CountdownCloseable countdownCloseable, CommandlineStreams streams, Object errorStreamLock) {
651         EventHandler<String> errConsumer = new NativeStdErrStreamConsumer(errorStreamLock);
652         LineConsumerThread stdErr = new LineConsumerThread(
653                 "fork-" + forkNumber + "-err-thread", streams.getStdErrChannel(), errConsumer, countdownCloseable);
654         stdErr.start();
655         return stdErr;
656     }
657 
658     private Iterable<Class<?>> getSuitesIterator() throws SurefireBooterForkException {
659         try {
660             AbstractPathConfiguration classpathConfiguration = startupConfiguration.getClasspathConfiguration();
661             ClassLoader unifiedClassLoader = classpathConfiguration.createMergedClassLoader();
662 
663             CommonReflector commonReflector = new CommonReflector(unifiedClassLoader);
664             Object reporterFactory = commonReflector.createReportingReporterFactory(startupReportConfiguration, log);
665 
666             ProviderFactory providerFactory = new ProviderFactory(
667                     startupConfiguration, providerConfiguration, unifiedClassLoader, reporterFactory);
668             SurefireProvider surefireProvider = providerFactory.createProvider(false);
669             return surefireProvider.getSuites();
670         } catch (SurefireExecutionException e) {
671             throw new SurefireBooterForkException("Unable to create classloader to find test suites", e);
672         }
673     }
674 
675     private static Thread createImmediateShutdownHookThread(
676             final TestLessInputStreamBuilder builder, final Shutdown shutdownType) {
677         return SHUTDOWN_HOOK_THREAD_FACTORY.newThread(new Runnable() {
678             @Override
679             public void run() {
680                 builder.getImmediateCommands().shutdown(shutdownType);
681             }
682         });
683     }
684 
685     private static Thread createCachableShutdownHookThread(
686             final TestLessInputStreamBuilder builder, final Shutdown shutdownType) {
687         return SHUTDOWN_HOOK_THREAD_FACTORY.newThread(new Runnable() {
688             @Override
689             public void run() {
690                 builder.getCachableCommands().shutdown(shutdownType);
691             }
692         });
693     }
694 
695     private static Thread createShutdownHookThread(
696             final Iterable<TestProvidingInputStream> streams, final Shutdown shutdownType) {
697         return SHUTDOWN_HOOK_THREAD_FACTORY.newThread(new Runnable() {
698             @Override
699             public void run() {
700                 for (TestProvidingInputStream stream : streams) {
701                     stream.shutdown(shutdownType);
702                 }
703             }
704         });
705     }
706 
707     private static ScheduledExecutorService createPingScheduler() {
708         ThreadFactory threadFactory = newDaemonThreadFactory("ping-timer-" + PING_IN_SECONDS + "s");
709         return newScheduledThreadPool(1, threadFactory);
710     }
711 
712     private static ScheduledExecutorService createTimeoutCheckScheduler() {
713         ThreadFactory threadFactory = newDaemonThreadFactory("timeout-check-timer");
714         return newScheduledThreadPool(1, threadFactory);
715     }
716 
717     private ScheduledFuture<?> triggerPingTimerForShutdown(final TestLessInputStreamBuilder builder) {
718         return pingThreadScheduler.scheduleWithFixedDelay(
719                 new Runnable() {
720                     @Override
721                     public void run() {
722                         builder.getImmediateCommands().noop();
723                     }
724                 },
725                 0,
726                 PING_IN_SECONDS,
727                 SECONDS);
728     }
729 
730     private ScheduledFuture<?> triggerPingTimerForShutdown(final Iterable<TestProvidingInputStream> streams) {
731         return pingThreadScheduler.scheduleWithFixedDelay(
732                 new Runnable() {
733                     @Override
734                     public void run() {
735                         for (TestProvidingInputStream stream : streams) {
736                             stream.noop();
737                         }
738                     }
739                 },
740                 0,
741                 PING_IN_SECONDS,
742                 SECONDS);
743     }
744 
745     private ScheduledFuture<?> triggerTimeoutCheck() {
746         return timeoutCheckScheduler.scheduleWithFixedDelay(
747                 new Runnable() {
748                     @Override
749                     public void run() {
750                         long systemTime = currentTimeMillis();
751                         for (ForkClient forkClient : currentForkClients) {
752                             forkClient.tryToTimeout(systemTime, forkedProcessTimeoutInSeconds);
753                         }
754                     }
755                 },
756                 0,
757                 TIMEOUT_CHECK_PERIOD_MILLIS,
758                 MILLISECONDS);
759     }
760 
761     private final class ForkedNodeArg implements ForkNodeArguments {
762         private final int forkChannelId;
763         private final File dumpLogDir;
764         private final String sessionId;
765         private final boolean debug;
766 
767         ForkedNodeArg(boolean debug, int forkChannelId, File dumpLogDir, String sessionId) {
768             this.debug = debug;
769             this.forkChannelId = forkChannelId;
770             this.dumpLogDir = dumpLogDir;
771             this.sessionId = sessionId;
772         }
773 
774         @Nonnull
775         @Override
776         public String getSessionId() {
777             return sessionId;
778         }
779 
780         @Override
781         public int getForkChannelId() {
782             return forkChannelId;
783         }
784 
785         @Override
786         @Nonnull
787         public File dumpStreamText(@Nonnull String text) {
788             return InPluginProcessDumpSingleton.getSingleton().dumpStreamText(text, dumpLogDir, forkChannelId);
789         }
790 
791         @Nonnull
792         @Override
793         public File dumpStreamException(@Nonnull Throwable t) {
794             return InPluginProcessDumpSingleton.getSingleton()
795                     .dumpStreamException(t, t.getLocalizedMessage(), dumpLogDir, forkChannelId);
796         }
797 
798         @Override
799         public void logWarningAtEnd(@Nonnull String text) {
800             logsAtEnd.add(text);
801         }
802 
803         @Nonnull
804         @Override
805         public ConsoleLogger getConsoleLogger() {
806             return log;
807         }
808 
809         @Nonnull
810         @Override
811         public Object getConsoleLock() {
812             return log;
813         }
814 
815         @Override
816         public File getEventStreamBinaryFile() {
817             return debug
818                     ? InPluginProcessDumpSingleton.getSingleton().getEventStreamBinaryFile(dumpLogDir, forkChannelId)
819                     : null;
820         }
821 
822         @Override
823         public File getCommandStreamBinaryFile() {
824             return null;
825         }
826     }
827 }