[TASK] bamboo 7.6 test specs and pgsql install test
[Packages/TYPO3.CMS.git] / Build / bamboo / src / main / java / core / AbstractCoreSpec.java
1 package core;
2
3 /*
4 * This file is part of the TYPO3 CMS project.
5 *
6 * It is free software; you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License, either version 2
8 * of the License, or any later version.
9 *
10 * For the full copyright and license information, please read the
11 * LICENSE.txt file that was distributed with this source code.
12 *
13 * The TYPO3 project - inspiring people to share!
14 */
15
16 import java.util.ArrayList;
17
18 import com.atlassian.bamboo.specs.api.builders.BambooKey;
19 import com.atlassian.bamboo.specs.api.builders.plan.Job;
20 import com.atlassian.bamboo.specs.api.builders.requirement.Requirement;
21 import com.atlassian.bamboo.specs.api.builders.task.Task;
22 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
23 import com.atlassian.bamboo.specs.builders.task.CommandTask;
24 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
25 import com.atlassian.bamboo.specs.builders.task.TestParserTask;
26 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
27 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
28 import com.atlassian.bamboo.specs.model.task.TestParserTaskProperties;
29
30 /**
31 * Abstract class with common methods of pre-merge and nightly plan
32 */
33 abstract public class AbstractCoreSpec {
34
35 protected String testingFrameworkBuildPath = "typo3/sysext/core/Build/";
36
37 protected String credentialsMysql =
38 "typo3DatabaseName=\"func\"" +
39 " typo3DatabaseUsername=\"funcu\"" +
40 " typo3DatabasePassword=\"funcp\"" +
41 " typo3DatabaseHost=\"localhost\"" +
42 " typo3InstallToolPassword=\"klaus\"";
43
44 /**
45 * Job composer validate
46 */
47 protected Job getJobComposerValidate() {
48 return new Job("Validate composer.json", new BambooKey("VC"))
49 .description("Validate composer.json before actual tests are executed")
50 .tasks(
51 this.getTaskGitCloneRepository(),
52 this.getTaskGitCherryPick(),
53 new CommandTask()
54 .description("composer validate")
55 .executable("composer").argument("validate")
56 );
57 }
58
59 /**
60 * Jobs for mysql based functional tests
61 *
62 * @param int numberOfChunks
63 * @param Requirement requirement
64 * @param String requirementIdentifier
65 */
66 protected ArrayList<Job> getJobsFunctionalTestsMysql(int numberOfChunks, Requirement requirement, String requirementIdentifier) {
67 ArrayList<Job> jobs = new ArrayList<Job>();
68
69 for (int i=0; i<numberOfChunks; i++) {
70 jobs.add(new Job("Func mysql " + requirementIdentifier + " 0" + i, new BambooKey("FMY" + requirementIdentifier + "0" + i))
71 .description("Run functional tests on mysql DB " + requirementIdentifier)
72 .tasks(
73 this.getTaskGitCloneRepository(),
74 this.getTaskGitCherryPick(),
75 this.getTaskComposerInstall(),
76 this.getTaskSplitFunctionalJobs(numberOfChunks),
77 new ScriptTask()
78 .description("Run phpunit with functional chunk 0" + i)
79 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
80 .inlineBody(
81 this.getScriptTaskBashInlineBody() +
82 "./bin/phpunit --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
83 )
84 .environmentVariables(this.credentialsMysql)
85 )
86 .finalTasks(
87 this.getTaskDeleteMysqlDatabases(),
88 new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
89 .resultDirectories("test-reports/phpunit.xml")
90 )
91 .requirements(
92 requirement
93 )
94 );
95 }
96
97 return jobs;
98 }
99
100 /**
101 * Job with various smaller script tests
102 */
103 protected Job getJobIntegrationVarious() {
104 // Exception code checker, xlf, permissions, rst file check
105 return new Job("Integration various", new BambooKey("CDECC"))
106 .description("Checks duplicate exceptions, git submodules, xlf files, permissions, rst")
107 .tasks(
108 this.getTaskGitCloneRepository(),
109 this.getTaskGitCherryPick(),
110 new ScriptTask()
111 .description("Run git submodule status and verify there are none")
112 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
113 .inlineBody(
114 this.getScriptTaskBashInlineBody() +
115 "if [[ `git submodule status 2>&1 | wc -l` -ne 0 ]]; then\n" +
116 " echo \\\"Found a submodule definition in repository\\\";\n" +
117 " exit 99;\n" +
118 "fi\n"
119 ),
120 new ScriptTask()
121 .description("Run xlf check")
122 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
123 .inlineBody(
124 this.getScriptTaskBashInlineBody() +
125 "./typo3/sysext/core/Build/Scripts/xlfcheck.sh"
126 )
127 );
128 }
129
130 /**
131 * Job for PHP lint
132 *
133 * @param Requirement requirement
134 * @param String requirementIdentfier
135 */
136 protected Job getJobLintPhp(Requirement requirement, String requirementIdentifier) {
137 return new Job("Lint " + requirementIdentifier, new BambooKey("L" + requirementIdentifier))
138 .description("Run php -l on source files for linting " + requirementIdentifier)
139 .tasks(
140 this.getTaskGitCloneRepository(),
141 this.getTaskGitCherryPick(),
142 new ScriptTask()
143 .description("Run php lint")
144 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
145 .inlineBody(
146 this.getScriptTaskBashInlineBody() +
147 "find . -name \\*.php -print0 | xargs -0 -n1 -P2 php -l >/dev/null\n"
148 )
149 )
150 .requirements(
151 requirement
152 );
153 }
154
155 /**
156 * Job for unit testing PHP
157 *
158 * @param Requirement requirement
159 * @param String requirementIdentfier
160 */
161 protected Job getJobUnitPhp(Requirement requirement, String requirementIdentifier) {
162 return new Job("Unit " + requirementIdentifier, new BambooKey("UT" + requirementIdentifier))
163 .description("Run unit tests " + requirementIdentifier)
164 .tasks(
165 this.getTaskGitCloneRepository(),
166 this.getTaskGitCherryPick(),
167 this.getTaskComposerInstall(),
168 new ScriptTask()
169 .description("Run phpunit")
170 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
171 .inlineBody(
172 this.getScriptTaskBashInlineBody() +
173 this.getScriptTaskBashPhpNoXdebug() +
174 "php_no_xdebug bin/phpunit --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "UnitTests.xml"
175 )
176 )
177 .finalTasks(
178 new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
179 .resultDirectories("test-reports/phpunit.xml")
180 )
181 .requirements(
182 requirement
183 );
184 }
185
186 /**
187 * Task definition for basic core clone of linked default repository
188 */
189 protected Task getTaskGitCloneRepository() {
190 return new VcsCheckoutTask()
191 .description("Checkout git core")
192 .checkoutItems(new CheckoutItem().defaultRepository())
193 .cleanCheckout(true);
194 }
195
196 /**
197 * Task definition to cherry pick a patch set from gerrit on top of cloned core
198 */
199 protected Task getTaskGitCherryPick() {
200 return new ScriptTask()
201 .description("Gerrit cherry pick")
202 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
203 .inlineBody(
204 this.getScriptTaskBashInlineBody() +
205 "CHANGEURL=${bamboo.changeUrl}\n" +
206 "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
207 "PATCHSET=${bamboo.patchset}\n" +
208 "\n" +
209 "if [[ $CHANGEURL ]]; then\n" +
210 " gerrit-cherry-pick https://review.typo3.org/Packages/TYPO3.CMS $CHANGEURLID/$PATCHSET || exit 1\n" +
211 "fi\n"
212 );
213 }
214
215 /**
216 * Task definition to execute composer install
217 */
218 protected Task getTaskComposerInstall() {
219 return new CommandTask()
220 .description("composer install")
221 .executable("composer")
222 .argument("install -n");
223 }
224
225 /**
226 * Task to delete any created mysql test databases, used as final task
227 */
228 protected Task getTaskDeleteMysqlDatabases() {
229 return new ScriptTask()
230 .description("Delete mysql test dbs")
231 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
232 .inlineBody(
233 this.getScriptTaskBashInlineBody() +
234 "DB_STARTS_WITH=\"func_\"\n" +
235 "MUSER=\"funcu\"\n" +
236 "MPWD=\"funcp\"\n" +
237 "MYSQL=\"mysql\"\n" +
238 "DBS=\"$($MYSQL -u $MUSER -p\"$MPWD\" -Bse 'show databases')\"\n" +
239 "\n" +
240 "for db in $DBS; do\n" +
241 " if [[ \"$db\" == $DB_STARTS_WITH* ]]; then\n" +
242 " echo \"Deleting $db\"\n" +
243 " $MYSQL -u $MUSER -p\"$MPWD\" -Bse \"drop database $db\"\n" +
244 " fi\n" +
245 "done\n"
246 );
247 }
248
249 /**
250 * Task to split functional jobs into chunks
251 */
252 protected Task getTaskSplitFunctionalJobs(int numberOfJobs) {
253 return new ScriptTask()
254 .description("Create list of test files to execute per job")
255 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
256 .inlineBody(
257 this.getScriptTaskBashInlineBody() +
258 "./" + this.testingFrameworkBuildPath + "Scripts/splitFunctionalTests.sh " + numberOfJobs
259 );
260 }
261
262 /**
263 * Requirement for php 5.5
264 */
265 protected Requirement getRequirementPhpVersion55() {
266 return new Requirement("system.phpVersion")
267 .matchValue("5.5")
268 .matchType(Requirement.MatchType.EQUALS);
269 }
270
271 /**
272 * Requirement for php 5.6
273 */
274 protected Requirement getRequirementPhpVersion56() {
275 return new Requirement("system.phpVersion")
276 .matchValue("5.6")
277 .matchType(Requirement.MatchType.EQUALS);
278 }
279
280 /**
281 * Requirement for php 7.0
282 */
283 protected Requirement getRequirementPhpVersion70() {
284 return new Requirement("system.phpVersion")
285 .matchValue("7.0")
286 .matchType(Requirement.MatchType.EQUALS);
287 }
288
289 /**
290 * Requirement for php 7.1
291 */
292 protected Requirement getRequirementPhpVersion71() {
293 return new Requirement("system.phpVersion")
294 .matchValue("7.1")
295 .matchType(Requirement.MatchType.EQUALS);
296 }
297
298 /**
299 * A bash header for script tasks forking a bash if needed
300 */
301 protected String getScriptTaskBashInlineBody() {
302 return
303 "#!/bin/bash\n" +
304 "\n" +
305 "if [ \"$(ps -p \"$$\" -o comm=)\" != \"bash\" ]; then\n" +
306 " bash \"$0\" \"$@\"\n" +
307 " exit \"$?\"\n" +
308 "fi\n" +
309 "\n";
310 }
311
312 /**
313 * A bash function providing a php bin without xdebug
314 */
315 protected String getScriptTaskBashPhpNoXdebug() {
316 return
317 "php_no_xdebug () {\n" +
318 " temporaryPath=\"$(mktemp -t php.XXXX).ini\"\n" +
319 " php -i | grep \"\\.ini\" | grep -o -e '\\(/[A-Za-z0-9._-]\\+\\)\\+\\.ini' | grep -v xdebug | xargs awk 'FNR==1{print \"\"}1' > \"${temporaryPath}\"\n" +
320 " php -n -c \"${temporaryPath}\" \"$@\"\n" +
321 " RETURN=$?\n" +
322 " rm -f \"${temporaryPath}\"\n" +
323 " exit $RETURN\n" +
324 "}\n" +
325 "\n";
326 }
327 }