Commit 730385ea authored by Christian Kuhn's avatar Christian Kuhn
Browse files

[TASK] runTests.sh: Allow chunking of acceptance and functional tasks

The '-c x/y' option for acceptance and functional jobs now
allows to split the set of acceptance/functional tests into
chuncks just like the CI does to share resources between
multiple parallel jobs.

Adding this to runTests.sh allows to replay the exact same
list of tests that CI does. This example will execute tests
of chunk 3 from 10 chunks with mariadb and php 7.4:

Build/Scripts/runTests.sh -s functional -d mariadb -p 7.4 -c 3/10

Resolves: #93361
Releases: master, 10.4, 9.5
Change-Id: I5753a8f83a3ebec6823cb0b44d4ca35f3a64055f
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/67528

Tested-by: default avatarTYPO3com <noreply@typo3.com>
Tested-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
Reviewed-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
parent 6df91f7d
......@@ -41,6 +41,7 @@ nbproject
/Build/testing-docker/local/.env
/typo3/sysext/*/Resources/Private/TypeScript/*.js
/typo3/sysext/*/Resources/Public/JavaScript/*.js.map
typo3/sysext/core/Tests/Acceptance/AcceptanceTests-Job-*
#
# TypeScript stuff
.baseDir.ts
......
......@@ -34,6 +34,9 @@ setUpDockerComposeDotEnv() {
echo "MYSQL_VERSION=${MYSQL_VERSION}" >> .env
echo "POSTGRES_VERSION=${POSTGRES_VERSION}" >> .env
echo "PHP_VERSION=${PHP_VERSION}" >> .env
echo "CHUNKS=${CHUNKS}" >> .env
echo "THISCHUNK=${THISCHUNK}" >> .env
# Set a custom database driver provided by option: -a
[[ ! -z "$DATABASE_DRIVER" ]] && echo "DATABASE_DRIVER=${DATABASE_DRIVER}" >> .env
}
......@@ -51,19 +54,6 @@ Usage: $0 [options] [file]
No arguments: Run all unit tests with PHP 7.2
Options:
-a <mysqli|pdo_mysql|sqlsrv|pdo_sqlsrv>
Only with -s functional
Specifies to use another driver, following combinations are available:
- mysql
- mysqli (default)
- pdo_mysql
- mariadb
- mysqli (default)
- pdo_mysql
- mssql
- sqlsrv (default)
- pdo_sqlsrv
-s <...>
Specifies which test suite to run
- acceptance: backend acceptance tests
......@@ -97,6 +87,19 @@ Options:
- unitJavascript: JavaScript unit tests
- unitRandom: PHP unit tests in random order, add -o <number> to use specific seed
-a <mysqli|pdo_mysql|sqlsrv|pdo_sqlsrv>
Only with -s functional
Specifies to use another driver, following combinations are available:
- mysql
- mysqli (default)
- pdo_mysql
- mariadb
- mysqli (default)
- pdo_mysql
- mssql
- sqlsrv (default)
- pdo_sqlsrv
-d <mariadb|mysql|mssql|postgres|sqlite>
Only with -s install|functional
Specifies on which DBMS tests are performed
......@@ -130,6 +133,11 @@ Options:
- 12
- 13
-c <chunk/numberOfChunks>
Only with -s functional|acceptance
Hack functional or acceptance tests into #numberOfChunks pieces and run tests of #chunk.
Example -c 3/13
-p <7.2|7.3|7.4|8.0>
Specifies the PHP minor version to be used
- 7.2 (default): use PHP 7.2
......@@ -241,6 +249,8 @@ CGLCHECK_DRY_RUN=""
MARIADB_VERSION="10.3"
MYSQL_VERSION="5.5"
POSTGRES_VERSION="10"
CHUNKS=0
THISCHUNK=0
# Option parsing
# Reset in case getopts has been used previously in the shell
......@@ -248,13 +258,22 @@ OPTIND=1
# Array for invalid options
INVALID_OPTIONS=();
# Simple option parsing based on getopts (! not getopt)
while getopts ":a:s:d:i:j:k::p:e:xy:o:nhuv" OPT; do
while getopts ":a:s:c:d:i:j:k:p:e:xy:o:nhuv" OPT; do
case ${OPT} in
s)
TEST_SUITE=${OPTARG}
;;
a)
DATABASE_DRIVER=${OPTARG}
;;
s)
TEST_SUITE=${OPTARG}
c)
if ! [[ ${OPTARG} =~ ^([0-9]+\/[0-9]+)$ ]]; then
INVALID_OPTIONS+=(${OPTARG})
else
# Split "2/13" - run chunk 2 of 13 chunks
THISCHUNK=`echo ${OPTARG} | cut -d '/' -f1`
CHUNKS=`echo ${OPTARG} | cut -d '/' -f2`
fi
;;
d)
DBMS=${OPTARG}
......@@ -343,6 +362,9 @@ fi
case ${TEST_SUITE} in
acceptance)
setUpDockerComposeDotEnv
if [ ${CHUNKS} -gt 1 ]; then
docker-compose run acceptance_split
fi
docker-compose run prepare_acceptance_backend_mariadb
docker-compose run acceptance_backend_mariadb
SUITE_EXIT_CODE=$?
......@@ -474,6 +496,9 @@ case ${TEST_SUITE} in
;;
functional)
setUpDockerComposeDotEnv
if [ ${CHUNKS} -gt 1 ]; then
docker-compose run functional_split
fi
case ${DBMS} in
mariadb)
[[ ! -z "$DATABASE_DRIVER" ]] && echo "Using driver: ${DATABASE_DRIVER}"
......
......@@ -48,6 +48,13 @@ services:
memcached1-5:
image: memcached:1.5-alpine
acceptance_split:
image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
user: ${HOST_UID}
volumes:
- ${CORE_ROOT}:${CORE_ROOT}
working_dir: ${CORE_ROOT}
command: php vendor/typo3/testing-framework/Resources/Core/Build/Scripts/splitAcceptanceTests.php -v ${CHUNKS}
prepare_acceptance_backend_mariadb:
image: alpine:3.8
......@@ -85,8 +92,15 @@ services:
if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
set -x
fi
mkdir -p typo3temp/var/tests/ \
&& ./bin/codecept run Backend -d -c typo3/sysext/core/Tests/codeception.yml ${TEST_FILE} --html reports.html
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"bin/codecept run Backend -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${TEST_FILE} --xml reports.xml --html reports.html\"
else
COMMAND=\"bin/codecept run Backend -d -c typo3/sysext/core/Tests/codeception.yml ${TEST_FILE} --xml reports.xml --html reports.html\"
fi
mkdir -p typo3temp/var/tests/
$${COMMAND}
"
prepare_acceptance_install_mariadb:
......@@ -546,6 +560,14 @@ services:
composer validate;
"
functional_split:
image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
user: ${HOST_UID}
volumes:
- ${CORE_ROOT}:${CORE_ROOT}
working_dir: ${CORE_ROOT}
command: php vendor/typo3/testing-framework/Resources/Core/Build/Scripts/splitFunctionalTests.php -v ${CHUNKS}
prepare_functional_mariadb:
image: alpine:3.8
links:
......@@ -586,13 +608,17 @@ services:
set -x
fi
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
else
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
fi
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
php -n -c /etc/php/cli-no-xdebug/php.ini \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
else
DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
fi
"
......@@ -636,13 +662,17 @@ services:
set -x
fi
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
else
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
fi
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
php -n -c /etc/php/cli-no-xdebug/php.ini \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
else
DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
fi
"
......@@ -689,13 +719,17 @@ services:
set -x
fi
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\"
else
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\"
fi
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
php -n -c /etc/php/cli-no-xdebug/php.ini \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
else
DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
fi
"
......@@ -739,13 +773,17 @@ services:
set -x
fi
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\"
else
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\"
fi
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
php -n -c /etc/php/cli-no-xdebug/php.ini \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE};
php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
else
DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE};
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
fi
"
......@@ -781,13 +819,17 @@ services:
set -x
fi
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\"
else
COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\"
fi
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
php -n -c /etc/php/cli-no-xdebug/php.ini \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE};
php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
else
DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE};
XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
fi
"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment