[TASK] Run acceptance tests in parallel 13/51713/7
authorTymoteusz Motylewski <t.motylewski@gmail.com>
Wed, 15 Feb 2017 22:40:42 +0000 (23:40 +0100)
committerChristian Kuhn <lolli@schwarzbu.ch>
Fri, 17 Feb 2017 00:44:22 +0000 (01:44 +0100)
Provide script for generating files with acceptance tests groups
to allow running them in parallel on different hosts. Very useful
for bamboo.

Usage:

Generating 5 dynamic group files:
./components/testing_framework/Resources/Core/Build/Scripts/splitAcceptanceTests.sh 5

Run tests of group 1:
./bin/codecept run Acceptance -g AcceptanceTests-Job-1 -c ...

Note group numbers start from 1, not 0.

Resolves: #79845
Releases: master
Change-Id: Ib74ec0b457ee927a906a77e648041ae15dc727d2
Reviewed-on: https://review.typo3.org/51713
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
components/testing_framework/Resources/Core/Build/AcceptanceTests.yml
components/testing_framework/Resources/Core/Build/Scripts/splitAcceptanceTests.sh [new file with mode: 0755]

index 923883b..15be77c 100644 (file)
@@ -14,3 +14,6 @@ extensions:
     - Codeception\Extension\RunFailed
     - Codeception\Extension\Recorder
     - TYPO3\TestingFramework\Core\Acceptance\AcceptanceCoreEnvironment
+groups:
+  AcceptanceTests-Job-*: Configuration/Acceptance/AcceptanceTests-Job-*
+
diff --git a/components/testing_framework/Resources/Core/Build/Scripts/splitAcceptanceTests.sh b/components/testing_framework/Resources/Core/Build/Scripts/splitAcceptanceTests.sh
new file mode 100755 (executable)
index 0000000..e7b51f7
--- /dev/null
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+#########################
+#
+# This file is typically executed by bamboo, but could be
+# used locally, too.
+#
+# It expects to be run from the core root.
+#
+# ./components/testing_framework/core/Build/Scripts/splitAcceptanceTests.sh <numberOfConfigs>
+#
+# The script finds all acceptance tests and creates <numberOfConfigs> number
+# of codeception group files, each containing a sub set of Cest files to execute.
+#
+# components/testing_framework/Resources/Core/Build/Configuration/Acceptance/AcceptanceTests-Job-<counter>
+#
+# Those sub-groups can then be executed with a command like (example for files in group 2 here)
+# ./bin/codecept run Acceptance -d -g AcceptanceTests-Job-2 -c components/testing_framework/Resources/Core/Build/AcceptanceTests.yml
+#
+#########################
+
+numberOfAcceptanceTestJobs=${1}
+
+# Have a dir for temp files and clean up possibly existing stuff
+if [ ! -d buildTemp ]; then
+       mkdir buildTemp || exit 1
+fi
+if [ -f buildTemp/testFiles.txt ]; then
+       rm buildTemp/testFiles.txt
+fi
+if [ -f buildTemp/testFilesWithNumberOfTestFiles.txt ]; then
+       rm buildTemp/testFilesWithNumberOfTestFiles.txt
+fi
+if [ -f buildTemp/testFilesWeighted.txt ]; then
+       rm buildTemp/testFilesWeighted.txt
+fi
+
+# A list of all acceptance test files
+find . -name \*Cest.php -path \*typo3/sysext/*/Tests/Acceptance* > buildTemp/testFiles.txt
+
+# File with test files of format "42 ./path/to/file"
+while read testFile; do
+       numberOfTestsInTestFile=`grep "public function [^\_].*" ${testFile} | wc -l`
+       echo "${numberOfTestsInTestFile} ${testFile}" >> buildTemp/testFilesWithNumberOfTestFiles.txt
+done < buildTemp/testFiles.txt
+
+# Sort list of files numeric
+cat buildTemp/testFilesWithNumberOfTestFiles.txt | sort -n -r > buildTemp/testFilesWeighted.txt
+
+groupFilePath="components/testing_framework/Resources/Core/Build/Configuration/Acceptance"
+# Config file boilerplate per job
+for (( i=1; i<=${numberOfAcceptanceTestJobs}; i++)); do
+       if [ -f ${groupFilePath}/AcceptanceTests-Job-${i} ]; then
+               rm ${groupFilePath}/AcceptanceTests-Job-${i}
+       fi
+       touch ${groupFilePath}/AcceptanceTests-Job-${i}
+done
+
+counter=0
+direction=ascending
+while read testFileWeighted; do
+       # test file only, without leading ./
+       testFile=`echo ${testFileWeighted} | cut -f2 -d" " | cut -f2-40 -d"/"`
+
+       # Goal: with 3 jobs, have:
+       # file #0 to job #0 (asc)
+       # file #1 to job #1 (asc)
+       # file #2 to job #2 (asc)
+       # file #3 to job #2 (desc)
+       # file #4 to job #1 (desc)
+       # file #5 to job #0 (desc)
+       # file #6 to job #0 (asc)
+       # ...
+       testFileModuleNumberOfJobs=$(( counter % numberOfAcceptanceTestJobs ))
+       if [[ ${direction} == "descending" ]]; then
+               targetJobNumberForFile=$(( numberOfAcceptanceTestJobs - testFileModuleNumberOfJobs))
+       else
+               targetJobNumberForFile=${testFileModuleNumberOfJobs}
+       fi
+       if [ ${testFileModuleNumberOfJobs} -eq ${numberOfAcceptanceTestJobs} ]; then
+               if [[ ${direction} == "descending" ]]; then
+                       direction=ascending
+               else
+                       direction=descending
+               fi
+       fi
+       echo "../../../../../${testFile}" >> ${groupFilePath}/AcceptanceTests-Job-$(( targetJobNumberForFile + 1 ))
+       (( counter ++ ))
+done < buildTemp/testFilesWeighted.txt
+
+# Clean up
+rm buildTemp/testFiles.txt
+rm buildTemp/testFilesWeighted.txt
+rm buildTemp/testFilesWithNumberOfTestFiles.txt
+rmdir buildTemp