[TASK] Ensure UTF-8 files contain no BOM
[Packages/TYPO3.CMS.git] / Build / Scripts / runTests.sh
1 #!/usr/bin/env bash
2
3 #
4 # TYPO3 core test runner based on docker and docker-compose.
5 #
6
7 # Function to write a .env file in Build/testing-docker/local
8 # This is read by docker-compose and vars defined here are
9 # used in Build/testing-docker/local/docker-compose.yml
10 setUpDockerComposeDotEnv() {
11 # Delete possibly existing local .env file if exists
12 [ -e .env ] && rm .env
13 # Set up a new .env file for docker-compose
14 echo "COMPOSE_PROJECT_NAME=local" >> .env
15 # To prevent access rights of files created by the testing, the docker image later
16 # runs with the same user that is currently executing the script. docker-compose can't
17 # use $UID directly itself since it is a shell variable and not an env variable, so
18 # we have to set it explicitly here.
19 echo "HOST_UID=`id -u`" >> .env
20 # Your local home directory for composer and npm caching
21 echo "HOST_HOME=${HOME}" >> .env
22 # Your local user
23 echo "CORE_ROOT"=${CORE_ROOT} >> .env
24 echo "HOST_USER=${USER}" >> .env
25 echo "TEST_FILE=${TEST_FILE}" >> .env
26 echo "PHP_XDEBUG_ON=${PHP_XDEBUG_ON}" >> .env
27 echo "PHP_XDEBUG_PORT=${PHP_XDEBUG_PORT}" >> .env
28 echo "DOCKER_PHP_IMAGE=${DOCKER_PHP_IMAGE}" >> .env
29 echo "EXTRA_TEST_OPTIONS=${EXTRA_TEST_OPTIONS}" >> .env
30 echo "SCRIPT_VERBOSE=${SCRIPT_VERBOSE}" >> .env
31 echo "PHPUNIT_RANDOM=${PHPUNIT_RANDOM}" >> .env
32 echo "CGLCHECK_DRY_RUN=${CGLCHECK_DRY_RUN}" >> .env
33 }
34
35 # Load help text into $HELP
36 read -r -d '' HELP <<EOF
37 TYPO3 core test runner. Execute acceptance, unit, functional and other test suites in
38 a docker based test environment. Handles execution of single test files, sending
39 xdebug information to a local IDE and more.
40
41 Successfully tested with docker version 18.06.1-ce and docker-compose 1.21.2.
42
43 Usage: $0 [options] [file]
44
45 No arguments: Run all unit tests with PHP 7.2
46
47 Options:
48 -s <...>
49 Specifies which test suite to run
50 - acceptance: backend acceptance tests
51 - buildCss: execute scss to css builder
52 - buildJavascript: execute typescript to javascript builder
53 - cglGit: test and fix latest committed patch for CGL compliance
54 - cglAll: test and fix all core php files
55 - checkAnnotations: check php code for allowed annotations
56 - checkBom: check UTF-8 files do not contain BOM
57 - checkComposer: check composer.json files for version integrity
58 - checkCsvFixtures: test integrity of functional test csv fixtures
59 - checkExceptionCodes: test core for duplicate exception codes
60 - checkExtensionScannerRst: test all .rst files referenced by extension scanner exist
61 - checkFilePathLength: test core file paths do not exceed maximum length
62 - checkGitSubmodule: test core git has no sub modules defined
63 - checkPermissions: test some core files for correct executable bits
64 - checkRst: test .rst files for integrity
65 - checkXlf: test .xlf files for integrity
66 - composerInstall: "composer install", handy if host has no PHP, uses composer cache of users home
67 - composerValidate: "composer validate"
68 - functional: functional tests
69 - install: installation acceptance tests, only with -d mariadb|postgres|sqlite
70 - lint: PHP linting
71 - lintScss: SCSS linting
72 - lintTypescript: TS linting
73 - unit (default): PHP unit tests
74 - unitDeprecated: deprecated PHP unit tests
75 - unitJavascript: JavaScript unit tests
76 - unitRandom: PHP unit tests in random order, add -o <number> to use specific seed
77
78 -d <mariadb|mssql|postgres|sqlite>
79 Only with -s install|functional
80 Specifies on which DBMS tests are performed
81 - mariadb (default): use mariadb
82 - mssql: use mssql microsoft sql server
83 - postgres: use postgres
84 - sqlite: use sqlite
85
86 -p <7.2|7.3>
87 Specifies the PHP minor version to be used
88 - 7.2 (default): use PHP 7.2
89 - 7.3: use PHP 7.3
90
91 -e "<phpunit or codeception options>"
92 Only with -s functional|unit|unitDeprecated|unitRandom
93 Additional options to send to phpunit (unit & functional tests) or codeception (acceptance
94 tests). For phpunit, options starting with "--" must be added after options starting with "-".
95 Example -e "-v --filter canRetrieveValueWithGP" to enable verbose output AND filter tests
96 named "canRetrieveValueWithGP"
97
98 -x
99 Only with -s functional|unit|unitDeprecated|unitRandom
100 Send information to host instance for test or system under test break points. This is especially
101 useful if a local PhpStorm instance is listening on default xdebug port 9000. A different port
102 can be selected with -y
103
104 -y <port>
105 Send xdebug information to a different port than default 9000 if an IDE like PhpStorm
106 is not listening on default port.
107
108 -o <number>
109 Only with -s unitRandom
110 Set specific random seed to replay a random run in this order again. The phpunit randomizer
111 outputs the used seed at the end (in bamboo core testing logs, too). Use that number to
112 replay the unit tests in that order.
113
114 -n
115 Only with -s cglGit|cglAll
116 Activate dry-run in CGL check that does not actively change files and only prints broken ones.
117
118 -u
119 Update existing typo3gmbh/phpXY:latest docker images. Maintenance call to docker pull latest
120 versions of the main php images. The images are updated once in a while and only the youngest
121 ones are supported by core testing. Use this if weird test errors occur. Also removes obsolete
122 image versions of typo3gmbh/phpXY.
123
124 -v
125 Enable verbose script output. Shows variables and docker commands.
126
127 -h
128 Show this help.
129
130 Examples:
131 # Run all core unit tests using PHP 7.2
132 ./Build/Scripts/runTests.sh
133 ./Build/Scripts/runTests.sh -s unit
134
135 # Run all core units tests and enable xdebug (have a PhpStorm listening on port 9000!)
136 ./Build/Scripts/runTests.sh -x
137
138 # Run unit tests in phpunit verbose mode with xdebug on PHP 7.3 and filter for test canRetrieveValueWithGP
139 ./Build/Scripts/runTests.sh -x -p 7.3 -e "-v --filter canRetrieveValueWithGP"
140
141 # Run unit tests with PHP 7.3 and have xdebug enabled
142 ./Build/Scripts/runTests.sh -x -p 7.3
143
144 # Run functional tests on postgres with xdebug, php 7.3 and execute a restricted set of tests
145 ./Build/Scripts/runTests.sh -x -p 7.3 -s functional -d postgres typo3/sysext/core/Tests/Functional/Authentication
146
147 # Run restricted set of backend acceptance tests
148 ./Build/Scripts/runTests.sh -s acceptance typo3/sysext/core/Tests/Acceptance/Backend/Login/BackendLoginCest.php
149
150 # Run installer tests of a new instance on sqlite
151 ./Build/Scripts/runTests.sh -s install -d sqlite
152 EOF
153
154 # Test if docker-compose exists, else exit out with error
155 if ! type "docker-compose" > /dev/null; then
156 echo "This script relies on docker and docker-compose. Please install" >&2
157 exit 1
158 fi
159
160 # Go to the directory this script is located, so everything else is relative
161 # to this dir, no matter from where this script is called.
162 THIS_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
163 cd "$THIS_SCRIPT_DIR" || exit 1
164
165 # Go to directory that contains the local docker-compose.yml file
166 cd ../testing-docker/local || exit 1
167
168 # Option defaults
169 CORE_ROOT="${PWD}/../../../"
170 TEST_SUITE="unit"
171 DBMS="mariadb"
172 PHP_VERSION="7.2"
173 PHP_XDEBUG_ON=0
174 PHP_XDEBUG_PORT=9000
175 EXTRA_TEST_OPTIONS=""
176 SCRIPT_VERBOSE=0
177 PHPUNIT_RANDOM="--order rand"
178 CGLCHECK_DRY_RUN=""
179
180 # Option parsing
181 # Reset in case getopts has been used previously in the shell
182 OPTIND=1
183 # Array for invalid options
184 INVALID_OPTIONS=();
185 # Simple option parsing based on getopts (! not getopt)
186 while getopts ":s:d:p:e:xy:o:nhuv" OPT; do
187 case ${OPT} in
188 s)
189 TEST_SUITE=${OPTARG}
190 ;;
191 d)
192 DBMS=${OPTARG}
193 ;;
194 p)
195 PHP_VERSION=${OPTARG}
196 ;;
197 e)
198 EXTRA_TEST_OPTIONS=${OPTARG}
199 ;;
200 x)
201 PHP_XDEBUG_ON=1
202 ;;
203 y)
204 PHP_XDEBUG_PORT=${OPTARG}
205 ;;
206 o)
207 PHPUNIT_RANDOM="--order :${OPTARG}"
208 ;;
209 n)
210 CGLCHECK_DRY_RUN="-n"
211 ;;
212 h)
213 echo "${HELP}"
214 exit 0
215 ;;
216 u)
217 TEST_SUITE=update
218 ;;
219 v)
220 SCRIPT_VERBOSE=1
221 ;;
222 \?)
223 INVALID_OPTIONS+=(${OPTARG})
224 ;;
225 :)
226 INVALID_OPTIONS+=(${OPTARG})
227 ;;
228 esac
229 done
230
231 # Exit on invalid options
232 if [ ${#INVALID_OPTIONS[@]} -ne 0 ]; then
233 echo "Invalid option(s):" >&2
234 for I in "${INVALID_OPTIONS[@]}"; do
235 echo "-"${I} >&2
236 done
237 echo >&2
238 echo "${HELP}" >&2
239 exit 1
240 fi
241
242 # Move "7.2" to "php72", the latter is the docker container name
243 DOCKER_PHP_IMAGE=`echo "php${PHP_VERSION}" | sed -e 's/\.//'`
244
245 # Set $1 to first mass argument, this is the optional test file or test directory to execute
246 shift $((OPTIND - 1))
247 TEST_FILE=${1}
248
249 if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
250 set -x
251 fi
252
253 # Suite execution
254 case ${TEST_SUITE} in
255 acceptance)
256 setUpDockerComposeDotEnv
257 docker-compose run prepare_acceptance_backend_mariadb10
258 docker-compose run acceptance_backend_mariadb10
259 SUITE_EXIT_CODE=$?
260 docker-compose down
261 ;;
262 buildCss)
263 setUpDockerComposeDotEnv
264 docker-compose run build_css
265 SUITE_EXIT_CODE=$?
266 docker-compose down
267 ;;
268 buildJavascript)
269 setUpDockerComposeDotEnv
270 docker-compose run build_javascript
271 SUITE_EXIT_CODE=$?
272 docker-compose down
273 ;;
274 cglGit)
275 setUpDockerComposeDotEnv
276 docker-compose run cgl_git
277 SUITE_EXIT_CODE=$?
278 docker-compose down
279 ;;
280 cglAll)
281 # Active dry-run for cglAll needs not "-n" but specific options
282 if [[ ! -z ${CGLCHECK_DRY_RUN} ]]; then
283 CGLCHECK_DRY_RUN="--dry-run --diff --diff-format udiff"
284 fi
285 setUpDockerComposeDotEnv
286 docker-compose run cgl_all
287 SUITE_EXIT_CODE=$?
288 docker-compose down
289 ;;
290 checkAnnotations)
291 setUpDockerComposeDotEnv
292 docker-compose run check_annotations
293 SUITE_EXIT_CODE=$?
294 docker-compose down
295 ;;
296 checkBom)
297 setUpDockerComposeDotEnv
298 docker-compose run check_bom
299 SUITE_EXIT_CODE=$?
300 docker-compose down
301 ;;
302 checkComposer)
303 setUpDockerComposeDotEnv
304 docker-compose run check_composer
305 SUITE_EXIT_CODE=$?
306 docker-compose down
307 ;;
308 checkCsvFixtures)
309 setUpDockerComposeDotEnv
310 docker-compose run check_csv_fixtures
311 SUITE_EXIT_CODE=$?
312 docker-compose down
313 ;;
314 checkExceptionCodes)
315 setUpDockerComposeDotEnv
316 docker-compose run check_exception_codes
317 SUITE_EXIT_CODE=$?
318 docker-compose down
319 ;;
320 checkExtensionScannerRst)
321 setUpDockerComposeDotEnv
322 docker-compose run check_extension_scanner_rst
323 SUITE_EXIT_CODE=$?
324 docker-compose down
325 ;;
326 checkFilePathLength)
327 setUpDockerComposeDotEnv
328 docker-compose run check_file_path_length
329 SUITE_EXIT_CODE=$?
330 docker-compose down
331 ;;
332 checkGitSubmodule)
333 setUpDockerComposeDotEnv
334 docker-compose run check_git_submodule
335 SUITE_EXIT_CODE=$?
336 docker-compose down
337 ;;
338 checkPermissions)
339 setUpDockerComposeDotEnv
340 docker-compose run check_permissions
341 SUITE_EXIT_CODE=$?
342 docker-compose down
343 ;;
344 checkRst)
345 setUpDockerComposeDotEnv
346 docker-compose run check_rst
347 SUITE_EXIT_CODE=$?
348 docker-compose down
349 ;;
350 checkXlf)
351 setUpDockerComposeDotEnv
352 docker-compose run check_xlf
353 SUITE_EXIT_CODE=$?
354 docker-compose down
355 ;;
356 composerInstall)
357 setUpDockerComposeDotEnv
358 docker-compose run composer_install
359 SUITE_EXIT_CODE=$?
360 docker-compose down
361 ;;
362 composerValidate)
363 setUpDockerComposeDotEnv
364 docker-compose run composer_validate
365 SUITE_EXIT_CODE=$?
366 docker-compose down
367 ;;
368 functional)
369 setUpDockerComposeDotEnv
370 case ${DBMS} in
371 mariadb)
372 docker-compose run prepare_functional_mariadb10
373 docker-compose run functional_mariadb10
374 SUITE_EXIT_CODE=$?
375 ;;
376 mssql)
377 docker-compose run prepare_functional_mssql2017cu9
378 docker-compose run functional_mssql2017cu9
379 SUITE_EXIT_CODE=$?
380 ;;
381 postgres)
382 docker-compose run prepare_functional_postgres10
383 docker-compose run functional_postgres10
384 SUITE_EXIT_CODE=$?
385 ;;
386 sqlite)
387 docker-compose run prepare_functional_sqlite
388 docker-compose run functional_sqlite
389 SUITE_EXIT_CODE=$?
390 ;;
391 *)
392 echo "Invalid -d option argument ${DBMS}" >&2
393 echo >&2
394 echo "${HELP}" >&2
395 exit 1
396 esac
397 docker-compose down
398 ;;
399 install)
400 setUpDockerComposeDotEnv
401 case ${DBMS} in
402 mariadb)
403 docker-compose run prepare_acceptance_install_mariadb10
404 docker-compose run acceptance_install_mariadb10
405 SUITE_EXIT_CODE=$?
406 ;;
407 postgres)
408 docker-compose run prepare_acceptance_install_postgres10
409 docker-compose run acceptance_install_postgres10
410 SUITE_EXIT_CODE=$?
411 ;;
412 sqlite)
413 docker-compose run prepare_acceptance_install_sqlite
414 docker-compose run acceptance_install_sqlite
415 SUITE_EXIT_CODE=$?
416 ;;
417 *)
418 echo "Invalid -d option argument ${DBMS}" >&2
419 echo >&2
420 echo "${HELP}" >&2
421 exit 1
422 esac
423 docker-compose down
424 ;;
425 lint)
426 setUpDockerComposeDotEnv
427 docker-compose run lint
428 SUITE_EXIT_CODE=$?
429 docker-compose down
430 ;;
431 lintScss)
432 setUpDockerComposeDotEnv
433 docker-compose run lint_scss
434 SUITE_EXIT_CODE=$?
435 docker-compose down
436 ;;
437 lintTypescript)
438 setUpDockerComposeDotEnv
439 docker-compose run lint_typescript
440 SUITE_EXIT_CODE=$?
441 docker-compose down
442 ;;
443 unit)
444 setUpDockerComposeDotEnv
445 docker-compose run unit
446 SUITE_EXIT_CODE=$?
447 docker-compose down
448 ;;
449 unitDeprecated)
450 setUpDockerComposeDotEnv
451 docker-compose run unitDeprecated
452 SUITE_EXIT_CODE=$?
453 docker-compose down
454 ;;
455 unitJavascript)
456 setUpDockerComposeDotEnv
457 docker-compose run unitJavascript
458 SUITE_EXIT_CODE=$?
459 docker-compose down
460 ;;
461 unitRandom)
462 setUpDockerComposeDotEnv
463 docker-compose run unitRandom
464 SUITE_EXIT_CODE=$?
465 docker-compose down
466 ;;
467 update)
468 # pull typo3gmbh/phpXY:latest versions of those ones that exist locally
469 docker images typo3gmbh/php*:latest --format "{{.Repository}}:latest" | xargs -I {} docker pull {}
470 # remove "dangling" typo3gmbh/phpXY images (those tagged as <none>)
471 docker images typo3gmbh/php* --filter "dangling=true" --format "{{.ID}}" | xargs -I {} docker rmi {}
472 ;;
473 *)
474 echo "Invalid -s option argument ${TEST_SUITE}" >&2
475 echo >&2
476 echo "${HELP}" >&2
477 exit 1
478 esac
479
480 exit $SUITE_EXIT_CODE