69d293f940a6fd60e974a54fc6a14ed1b5640f8b
[Packages/TYPO3.CMS.git] / Build / bamboo / src / main / java / core / AbstractCoreSpec.java
1 package core;
2
3 /*
4 * This file is part of the TYPO3 CMS project.
5 *
6 * It is free software; you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License, either version 2
8 * of the License, or any later version.
9 *
10 * For the full copyright and license information, please read the
11 * LICENSE.txt file that was distributed with this source code.
12 *
13 * The TYPO3 project - inspiring people to share!
14 */
15
16 import java.util.ArrayList;
17
18 import com.atlassian.bamboo.specs.api.builders.BambooKey;
19 import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
20 import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
21 import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
22 import com.atlassian.bamboo.specs.api.builders.plan.Job;
23 import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
24 import com.atlassian.bamboo.specs.api.builders.plan.configuration.AllOtherPluginsConfiguration;
25 import com.atlassian.bamboo.specs.api.builders.plan.configuration.PluginConfiguration;
26 import com.atlassian.bamboo.specs.api.builders.requirement.Requirement;
27 import com.atlassian.bamboo.specs.api.builders.task.Task;
28 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
29 import com.atlassian.bamboo.specs.builders.task.CommandTask;
30 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
31 import com.atlassian.bamboo.specs.builders.task.TestParserTask;
32 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
33 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
34 import com.atlassian.bamboo.specs.model.task.TestParserTaskProperties;
35 import com.atlassian.bamboo.specs.util.MapBuilder;
36
37 /**
38 * Abstract class with common methods of pre-merge and nightly plan
39 */
40 abstract public class AbstractCoreSpec {
41
42 protected static String bambooServerName = "https://bamboo.typo3.com:443";
43 protected static String projectName = "TYPO3 Core";
44 protected static String projectKey = "CORE";
45
46 protected String testingFrameworkBuildPath = "typo3/sysext/core/Build/";
47
48 protected String credentialsMysql =
49 "typo3DatabaseName=\"func\"" +
50 " typo3DatabaseUsername=\"funcu\"" +
51 " typo3DatabasePassword=\"funcp\"" +
52 " typo3DatabaseHost=\"localhost\"" +
53 " typo3InstallToolPassword=\"klaus\"";
54
55 /**
56 * Default permissions on core plans
57 *
58 * @param projectName
59 * @param planName
60 * @return
61 */
62 protected PlanPermissions getDefaultPlanPermissions(String projectKey, String planKey) {
63 return new PlanPermissions(new PlanIdentifier(projectKey, planKey))
64 .permissions(new Permissions()
65 .groupPermissions("TYPO3 GmbH", PermissionType.ADMIN, PermissionType.VIEW, PermissionType.EDIT, PermissionType.BUILD, PermissionType.CLONE)
66 .groupPermissions("TYPO3 Core Team", PermissionType.VIEW, PermissionType.BUILD)
67 .loggedInUserPermissions(PermissionType.VIEW)
68 .anonymousUserPermissionView()
69 );
70 }
71
72 /**
73 * Default plan plugin configuration
74 *
75 * @return
76 */
77 protected PluginConfiguration getDefaultPlanPluginConfiguration() {
78 return new AllOtherPluginsConfiguration()
79 .configuration(new MapBuilder()
80 .put("custom", new MapBuilder()
81 .put("artifactHandlers.useCustomArtifactHandlers", "false")
82 .put("buildExpiryConfig", new MapBuilder()
83 .put("duration", "30")
84 .put("period", "days")
85 .put("labelsToKeep", "")
86 .put("expiryTypeResult", "true")
87 .put("buildsToKeep", "")
88 .put("enabled", "true")
89 .build()
90 )
91 .build()
92 )
93 .build()
94 );
95 }
96
97 /**
98 * Default job plugin configuration
99 *
100 * @return
101 */
102 protected PluginConfiguration getDefaultJobPluginConfiguration() {
103 return new AllOtherPluginsConfiguration()
104 .configuration(new MapBuilder()
105 .put("repositoryDefiningWorkingDirectory", -1)
106 .put("custom", new MapBuilder()
107 .put("auto", new MapBuilder()
108 .put("regex", "")
109 .put("label", "")
110 .build()
111 )
112 .put("buildHangingConfig.enabled", "false")
113 .put("ncover.path", "")
114 .put("clover", new MapBuilder()
115 .put("path", "")
116 .put("license", "")
117 .put("useLocalLicenseKey", "true")
118 .build()
119 )
120 .build()
121 )
122 .build()
123 );
124 }
125
126 /**
127 * Job composer validate
128 */
129 protected Job getJobComposerValidate() {
130 return new Job("Validate composer.json", new BambooKey("VC"))
131 .description("Validate composer.json before actual tests are executed")
132 .pluginConfigurations(this.getDefaultJobPluginConfiguration())
133 .tasks(
134 this.getTaskGitCloneRepository(),
135 this.getTaskGitCherryPick(),
136 new CommandTask()
137 .description("composer validate")
138 .executable("composer").argument("validate")
139 )
140 .cleanWorkingDirectory(true);
141 }
142
143 /**
144 * Jobs for mysql based functional tests
145 *
146 * @param int numberOfChunks
147 * @param Requirement requirement
148 * @param String requirementIdentifier
149 */
150 protected ArrayList<Job> getJobsFunctionalTestsMysql(int numberOfChunks, Requirement requirement, String requirementIdentifier) {
151 ArrayList<Job> jobs = new ArrayList<Job>();
152
153 for (int i=0; i<numberOfChunks; i++) {
154 jobs.add(new Job("Func mysql " + requirementIdentifier + " 0" + i, new BambooKey("FMY" + requirementIdentifier + "0" + i))
155 .description("Run functional tests on mysql DB " + requirementIdentifier)
156 .pluginConfigurations(this.getDefaultJobPluginConfiguration())
157 .tasks(
158 this.getTaskGitCloneRepository(),
159 this.getTaskGitCherryPick(),
160 this.getTaskComposerInstall(),
161 this.getTaskSplitFunctionalJobs(numberOfChunks),
162 new ScriptTask()
163 .description("Run phpunit with functional chunk 0" + i)
164 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
165 .inlineBody(
166 this.getScriptTaskBashInlineBody() +
167 "./bin/phpunit --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
168 )
169 .environmentVariables(this.credentialsMysql)
170 )
171 .finalTasks(
172 this.getTaskDeleteMysqlDatabases(),
173 new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
174 .resultDirectories("test-reports/phpunit.xml")
175 )
176 .requirements(
177 requirement
178 )
179 .cleanWorkingDirectory(true)
180 );
181 }
182
183 return jobs;
184 }
185
186 /**
187 * Job with various smaller script tests
188 */
189 protected Job getJobIntegrationVarious() {
190 // Exception code checker, xlf, permissions, rst file check
191 return new Job("Integration various", new BambooKey("CDECC"))
192 .description("Checks duplicate exceptions, git submodules, xlf files, permissions, rst")
193 .pluginConfigurations(this.getDefaultJobPluginConfiguration())
194 .tasks(
195 this.getTaskGitCloneRepository(),
196 this.getTaskGitCherryPick(),
197 new ScriptTask()
198 .description("Run git submodule status and verify there are none")
199 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
200 .inlineBody(
201 this.getScriptTaskBashInlineBody() +
202 "if [[ `git submodule status 2>&1 | wc -l` -ne 0 ]]; then\n" +
203 " echo \\\"Found a submodule definition in repository\\\";\n" +
204 " exit 99;\n" +
205 "fi\n"
206 ),
207 new ScriptTask()
208 .description("Run xlf check")
209 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
210 .inlineBody(
211 this.getScriptTaskBashInlineBody() +
212 "./typo3/sysext/core/Build/Scripts/xlfcheck.sh"
213 )
214 )
215 .cleanWorkingDirectory(true);
216 }
217
218 /**
219 * Job for PHP lint
220 *
221 * @param Requirement requirement
222 * @param String requirementIdentfier
223 */
224 protected Job getJobLintPhp(Requirement requirement, String requirementIdentifier) {
225 return new Job("Lint " + requirementIdentifier, new BambooKey("L" + requirementIdentifier))
226 .description("Run php -l on source files for linting " + requirementIdentifier)
227 .pluginConfigurations(this.getDefaultJobPluginConfiguration())
228 .tasks(
229 this.getTaskGitCloneRepository(),
230 this.getTaskGitCherryPick(),
231 new ScriptTask()
232 .description("Run php lint")
233 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
234 .inlineBody(
235 this.getScriptTaskBashInlineBody() +
236 "find . -name \\*.php -print0 | xargs -0 -n1 -P2 php -l >/dev/null\n"
237 )
238 )
239 .requirements(
240 requirement
241 )
242 .cleanWorkingDirectory(true);
243 }
244
245 /**
246 * Job for unit testing PHP
247 *
248 * @param Requirement requirement
249 * @param String requirementIdentfier
250 */
251 protected Job getJobUnitPhp(Requirement requirement, String requirementIdentifier) {
252 return new Job("Unit " + requirementIdentifier, new BambooKey("UT" + requirementIdentifier))
253 .description("Run unit tests " + requirementIdentifier)
254 .pluginConfigurations(this.getDefaultJobPluginConfiguration())
255 .tasks(
256 this.getTaskGitCloneRepository(),
257 this.getTaskGitCherryPick(),
258 this.getTaskComposerInstall(),
259 new ScriptTask()
260 .description("Run phpunit")
261 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
262 .inlineBody(
263 this.getScriptTaskBashInlineBody() +
264 this.getScriptTaskBashPhpNoXdebug() +
265 "php_no_xdebug bin/phpunit --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "UnitTests.xml"
266 )
267 )
268 .finalTasks(
269 new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
270 .resultDirectories("test-reports/phpunit.xml")
271 )
272 .requirements(
273 requirement
274 )
275 .cleanWorkingDirectory(true);
276 }
277
278 /**
279 * Task definition for basic core clone of linked default repository
280 */
281 protected Task getTaskGitCloneRepository() {
282 return new VcsCheckoutTask()
283 .description("Checkout git core")
284 .checkoutItems(new CheckoutItem().defaultRepository())
285 .cleanCheckout(true);
286 }
287
288 /**
289 * Task definition to cherry pick a patch set from gerrit on top of cloned core
290 */
291 protected Task getTaskGitCherryPick() {
292 return new ScriptTask()
293 .description("Gerrit cherry pick")
294 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
295 .inlineBody(
296 this.getScriptTaskBashInlineBody() +
297 "CHANGEURL=${bamboo.changeUrl}\n" +
298 "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
299 "PATCHSET=${bamboo.patchset}\n" +
300 "\n" +
301 "if [[ $CHANGEURL ]]; then\n" +
302 " gerrit-cherry-pick https://review.typo3.org/Packages/TYPO3.CMS $CHANGEURLID/$PATCHSET || exit 1\n" +
303 "fi\n"
304 );
305 }
306
307 /**
308 * Task definition to execute composer install
309 */
310 protected Task getTaskComposerInstall() {
311 return new CommandTask()
312 .description("composer install")
313 .executable("composer")
314 .argument("install -n");
315 }
316
317 /**
318 * Task to delete any created mysql test databases, used as final task
319 */
320 protected Task getTaskDeleteMysqlDatabases() {
321 return new ScriptTask()
322 .description("Delete mysql test dbs")
323 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
324 .inlineBody(
325 this.getScriptTaskBashInlineBody() +
326 "DB_STARTS_WITH=\"func_\"\n" +
327 "MUSER=\"funcu\"\n" +
328 "MPWD=\"funcp\"\n" +
329 "MYSQL=\"mysql\"\n" +
330 "DBS=\"$($MYSQL -u $MUSER -p\"$MPWD\" -Bse 'show databases')\"\n" +
331 "\n" +
332 "for db in $DBS; do\n" +
333 " if [[ \"$db\" == $DB_STARTS_WITH* ]]; then\n" +
334 " echo \"Deleting $db\"\n" +
335 " $MYSQL -u $MUSER -p\"$MPWD\" -Bse \"drop database $db\"\n" +
336 " fi\n" +
337 "done\n"
338 );
339 }
340
341 /**
342 * Task to split functional jobs into chunks
343 */
344 protected Task getTaskSplitFunctionalJobs(int numberOfJobs) {
345 return new ScriptTask()
346 .description("Create list of test files to execute per job")
347 .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
348 .inlineBody(
349 this.getScriptTaskBashInlineBody() +
350 "./" + this.testingFrameworkBuildPath + "Scripts/splitFunctionalTests.sh " + numberOfJobs
351 );
352 }
353
354 /**
355 * Requirement for php 5.5
356 */
357 protected Requirement getRequirementPhpVersion55() {
358 return new Requirement("system.phpVersion")
359 .matchValue("5.5")
360 .matchType(Requirement.MatchType.EQUALS);
361 }
362
363 /**
364 * Requirement for php 5.6
365 */
366 protected Requirement getRequirementPhpVersion56() {
367 return new Requirement("system.phpVersion")
368 .matchValue("5.6")
369 .matchType(Requirement.MatchType.EQUALS);
370 }
371
372 /**
373 * Requirement for php 7.0
374 */
375 protected Requirement getRequirementPhpVersion70() {
376 return new Requirement("system.phpVersion")
377 .matchValue("7.0")
378 .matchType(Requirement.MatchType.EQUALS);
379 }
380
381 /**
382 * Requirement for php 7.1
383 */
384 protected Requirement getRequirementPhpVersion71() {
385 return new Requirement("system.phpVersion")
386 .matchValue("7.1")
387 .matchType(Requirement.MatchType.EQUALS);
388 }
389
390 /**
391 * A bash header for script tasks forking a bash if needed
392 */
393 protected String getScriptTaskBashInlineBody() {
394 return
395 "#!/bin/bash\n" +
396 "\n" +
397 "if [ \"$(ps -p \"$$\" -o comm=)\" != \"bash\" ]; then\n" +
398 " bash \"$0\" \"$@\"\n" +
399 " exit \"$?\"\n" +
400 "fi\n" +
401 "\n";
402 }
403
404 /**
405 * A bash function providing a php bin without xdebug
406 */
407 protected String getScriptTaskBashPhpNoXdebug() {
408 return
409 "php_no_xdebug () {\n" +
410 " temporaryPath=\"$(mktemp -t php.XXXX).ini\"\n" +
411 " php -i | grep \"\\.ini\" | grep -o -e '\\(/[A-Za-z0-9._-]\\+\\)\\+\\.ini' | grep -v xdebug | xargs awk 'FNR==1{print \"\"}1' > \"${temporaryPath}\"\n" +
412 " php -n -c \"${temporaryPath}\" \"$@\"\n" +
413 " RETURN=$?\n" +
414 " rm -f \"${temporaryPath}\"\n" +
415 " exit $RETURN\n" +
416 "}\n" +
417 "\n";
418 }
419 }