Commit b9754f67 authored by Jochen Roth's avatar Jochen Roth Committed by Christian Kuhn
Browse files

[TASK] Add acceptance test reports to Gitlab

Gitlab is able to give a nice overview of tests
and how often a test failed recently which
makes it easier to investigate flaky/failed
tests. Also it takes away the pain of searching
for the actual failing test in a job because it only
displays information of the failed test
(no bootstrap log details and such)

Codeception AC test reports have been added to the
Gitlab GUI using the --xml flag to generate a Gitlab
readable report format and it has been added to the
jobs artifacts:reports:junit.

Resolves: #96628
Releases: main, 11.5
Change-Id: Id5ab5adfdc318c0c97143b0f05311f47fd1d37f0
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/73136


Tested-by: core-ci's avatarcore-ci <typo3@b13.com>
Tested-by: Oliver Bartsch's avatarOliver Bartsch <bo@cedev.de>
Tested-by: Stefan Bürk's avatarStefan Bürk <stefan@buerk.tech>
Tested-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
Reviewed-by: Oliver Bartsch's avatarOliver Bartsch <bo@cedev.de>
Reviewed-by: Stefan Bürk's avatarStefan Bürk <stefan@buerk.tech>
Reviewed-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
parent 883f3a3b
......@@ -8,6 +8,9 @@ acceptance application mariadb php 8.1 locked:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
......@@ -26,6 +29,9 @@ acceptance application mariadb php 8.1 max:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
......@@ -44,6 +50,9 @@ acceptance application mariadb php 8.1 min:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
......@@ -59,6 +68,9 @@ acceptance application mysql php 8.1 locked:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
......@@ -77,6 +89,9 @@ acceptance application mysql php 8.1 max:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
......@@ -95,6 +110,9 @@ acceptance application mysql php 8.1 min:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
......@@ -110,6 +128,9 @@ acceptance application sqlite php 8.1 locked:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
......@@ -128,6 +149,9 @@ acceptance application sqlite php 8.1 max:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
......@@ -146,6 +170,9 @@ acceptance application sqlite php 8.1 min:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 8
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
......
......@@ -7,6 +7,9 @@ acceptance install mariadb php 8.1 locked:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mariadb -i 10.3 -p 8.1
......@@ -39,6 +42,9 @@ acceptance install mariadb php 8.1 min:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mariadb -i 10.3 -p 8.1
......@@ -52,6 +58,9 @@ acceptance install mysql php 8.1 locked:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
......@@ -68,6 +77,9 @@ acceptance install mysql php 8.1 max:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
......@@ -84,6 +96,9 @@ acceptance install mysql php 8.1 min:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
......@@ -97,6 +112,9 @@ acceptance install postgres php 8.1 locked:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
......@@ -113,6 +131,9 @@ acceptance install postgres php 8.1 max:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
......@@ -129,6 +150,9 @@ acceptance install postgres php 8.1 min:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
......@@ -142,6 +166,9 @@ acceptance install sqlite php 8.1 locked:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
......@@ -158,6 +185,9 @@ acceptance install sqlite php 8.1 max:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
......@@ -174,6 +204,9 @@ acceptance install sqlite php 8.1 min:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
......@@ -9,6 +9,9 @@ acceptance application mariadb php 8.1 pre-merge:
paths:
- typo3temp/var/tests/acceptance/typo3temp/var/log
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
parallel: 13
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
......
......@@ -8,6 +8,9 @@ acceptance install mariadb php 8.1 pre-merge:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mariadb -i 10.3 -p 8.1
......@@ -22,6 +25,9 @@ acceptance install mysql php 8.1 pre-merge:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
......@@ -36,6 +42,9 @@ acceptance install postgres php 8.1 pre-merge:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
......@@ -50,6 +59,9 @@ acceptance install sqlite php 8.1 pre-merge:
when: on_failure
paths:
- typo3temp/var/tests/AcceptanceReports
reports:
junit:
- typo3temp/var/tests/AcceptanceReports/reports.xml
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
......@@ -100,9 +100,9 @@ services:
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
else
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
fi
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
......@@ -154,9 +154,9 @@ services:
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
else
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
fi
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
......@@ -211,9 +211,9 @@ services:
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
else
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
fi
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
......@@ -257,9 +257,9 @@ services:
php -v | grep '^PHP'
if [ ${CHUNKS} -gt 0 ]; then
echo \"Running chunk ${THISCHUNK}\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
else
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
fi
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
......@@ -311,12 +311,12 @@ services:
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
XDEBUG_MODE=\"off\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --xml reports.xml --html reports.html
else
XDEBUG_MODE=\"debug,develop\" \
XDEBUG_TRIGGER=\"foo\" \
XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --xml reports.xml --html reports.html
fi
"
......@@ -358,12 +358,12 @@ services:
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
XDEBUG_MODE=\"off\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --xml reports.xml --html reports.html
else
XDEBUG_MODE=\"debug,develop\" \
XDEBUG_TRIGGER=\"foo\" \
XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --xml reports.xml --html reports.html
fi
"
......@@ -405,12 +405,12 @@ services:
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
XDEBUG_MODE=\"off\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --xml reports.xml --html reports.html
else
XDEBUG_MODE=\"debug,develop\" \
XDEBUG_TRIGGER=\"foo\" \
XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --xml reports.xml --html reports.html
fi
"
......@@ -439,12 +439,12 @@ services:
mkdir -p typo3temp/var/tests/
if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
XDEBUG_MODE=\"off\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --xml reports.xml --html reports.html
else
XDEBUG_MODE=\"debug,develop\" \
XDEBUG_TRIGGER=\"foo\" \
XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --html reports.html
bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --xml reports.xml --html reports.html
fi
"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment