[TASK] bamboo 7.6 test specs and pgsql install test 19/53319/2
authorChristian Kuhn <lolli@schwarzbu.ch>
Thu, 15 Jun 2017 17:08:27 +0000 (19:08 +0200)
committerChristian Kuhn <lolli@schwarzbu.ch>
Fri, 23 Jun 2017 20:24:18 +0000 (22:24 +0200)
Backport of the bamboo spec patches from master to
7.6 to manage the build plan.

Change-Id: Icd4c84fef396a1745e72be4fc8396284fbe1afe4
Resolves: #81589
Resolves: #81660
Resolves: #81683
Releases: 7.6
Reviewed-on: https://review.typo3.org/53319
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
Build/bamboo/.credentials.example [new file with mode: 0644]
Build/bamboo/.gitignore [new file with mode: 0644]
Build/bamboo/pom.xml [new file with mode: 0644]
Build/bamboo/src/main/java/core/AbstractCoreSpec.java [new file with mode: 0644]
Build/bamboo/src/main/java/core/PreMergeSpec.java [new file with mode: 0644]
Build/bamboo/src/test/java/core/PreMergeSpecTest.java [new file with mode: 0644]

diff --git a/Build/bamboo/.credentials.example b/Build/bamboo/.credentials.example
new file mode 100644 (file)
index 0000000..7411d52
--- /dev/null
@@ -0,0 +1,2 @@
+username=admin
+password=aPassword
diff --git a/Build/bamboo/.gitignore b/Build/bamboo/.gitignore
new file mode 100644 (file)
index 0000000..52a69c6
--- /dev/null
@@ -0,0 +1,36 @@
+.credentials
+
+### Maven
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+
+### Java
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# Package Files #
+*.jar
+
+### Never add private credentials file to upstream
+.credentials
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+### Idea
+.idea/
+*.iml
+
+### Eclipse
+.classpath
+.project
+.settings/
diff --git a/Build/bamboo/pom.xml b/Build/bamboo/pom.xml
new file mode 100644 (file)
index 0000000..a47fddd
--- /dev/null
@@ -0,0 +1,36 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>com.atlassian.bamboo</groupId>
+    <artifactId>bamboo-specs-parent</artifactId>
+    <version>6.0.2</version>
+    <relativePath/>
+  </parent>
+
+  <groupId>com.atlassian.bamboo</groupId>
+  <artifactId>typo3-core</artifactId>
+  <version>1.0.0</version>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.atlassian.bamboo</groupId>
+      <artifactId>bamboo-specs-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.atlassian.bamboo</groupId>
+      <artifactId>bamboo-specs</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <!-- run 'mvn test' to perform offline validation of the plan -->
+  <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
+</project>
diff --git a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
new file mode 100644 (file)
index 0000000..25d718f
--- /dev/null
@@ -0,0 +1,327 @@
+package core;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+import java.util.ArrayList;
+
+import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.plan.Job;
+import com.atlassian.bamboo.specs.api.builders.requirement.Requirement;
+import com.atlassian.bamboo.specs.api.builders.task.Task;
+import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
+import com.atlassian.bamboo.specs.builders.task.CommandTask;
+import com.atlassian.bamboo.specs.builders.task.ScriptTask;
+import com.atlassian.bamboo.specs.builders.task.TestParserTask;
+import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
+import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
+import com.atlassian.bamboo.specs.model.task.TestParserTaskProperties;
+
+/**
+ * Abstract class with common methods of pre-merge and nightly plan
+ */
+abstract public class AbstractCoreSpec {
+
+    protected String testingFrameworkBuildPath = "typo3/sysext/core/Build/";
+
+    protected String credentialsMysql =
+        "typo3DatabaseName=\"func\"" +
+        " typo3DatabaseUsername=\"funcu\"" +
+        " typo3DatabasePassword=\"funcp\"" +
+        " typo3DatabaseHost=\"localhost\"" +
+        " typo3InstallToolPassword=\"klaus\"";
+
+    /**
+     * Job composer validate
+     */
+    protected Job getJobComposerValidate() {
+        return new Job("Validate composer.json", new BambooKey("VC"))
+        .description("Validate composer.json before actual tests are executed")
+        .tasks(
+            this.getTaskGitCloneRepository(),
+            this.getTaskGitCherryPick(),
+            new CommandTask()
+                .description("composer validate")
+                .executable("composer").argument("validate")
+        );
+    }
+
+    /**
+     * Jobs for mysql based functional tests
+     *
+     * @param int numberOfChunks
+     * @param Requirement requirement
+     * @param String requirementIdentifier
+     */
+    protected ArrayList<Job> getJobsFunctionalTestsMysql(int numberOfChunks, Requirement requirement, String requirementIdentifier) {
+        ArrayList<Job> jobs = new ArrayList<Job>();
+
+        for (int i=0; i<numberOfChunks; i++) {
+            jobs.add(new Job("Func mysql " + requirementIdentifier + " 0" + i, new BambooKey("FMY" + requirementIdentifier + "0" + i))
+                .description("Run functional tests on mysql DB " + requirementIdentifier)
+                .tasks(
+                    this.getTaskGitCloneRepository(),
+                    this.getTaskGitCherryPick(),
+                    this.getTaskComposerInstall(),
+                    this.getTaskSplitFunctionalJobs(numberOfChunks),
+                    new ScriptTask()
+                        .description("Run phpunit with functional chunk 0" + i)
+                        .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                        .inlineBody(
+                            this.getScriptTaskBashInlineBody() +
+                            "./bin/phpunit --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
+                        )
+                        .environmentVariables(this.credentialsMysql)
+                )
+                .finalTasks(
+                    this.getTaskDeleteMysqlDatabases(),
+                    new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
+                        .resultDirectories("test-reports/phpunit.xml")
+                )
+                .requirements(
+                    requirement
+                )
+            );
+        }
+
+        return jobs;
+    }
+
+    /**
+     * Job with various smaller script tests
+     */
+    protected Job getJobIntegrationVarious() {
+        // Exception code checker, xlf, permissions, rst file check
+        return new Job("Integration various", new BambooKey("CDECC"))
+            .description("Checks duplicate exceptions, git submodules, xlf files, permissions, rst")
+            .tasks(
+                this.getTaskGitCloneRepository(),
+                this.getTaskGitCherryPick(),
+                new ScriptTask()
+                    .description("Run git submodule status and verify there are none")
+                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                    .inlineBody(
+                        this.getScriptTaskBashInlineBody() +
+                        "if [[ `git submodule status 2>&1 | wc -l` -ne 0 ]]; then\n" +
+                        "    echo \\\"Found a submodule definition in repository\\\";\n" +
+                        "    exit 99;\n" +
+                        "fi\n"
+                    ),
+                new ScriptTask()
+                    .description("Run xlf check")
+                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                    .inlineBody(
+                        this.getScriptTaskBashInlineBody() +
+                        "./typo3/sysext/core/Build/Scripts/xlfcheck.sh"
+                    )
+            );
+    }
+
+    /**
+     * Job for PHP lint
+     *
+     * @param Requirement requirement
+     * @param String requirementIdentfier
+     */
+    protected Job getJobLintPhp(Requirement requirement, String requirementIdentifier) {
+        return new Job("Lint " + requirementIdentifier, new BambooKey("L" + requirementIdentifier))
+            .description("Run php -l on source files for linting " + requirementIdentifier)
+            .tasks(
+                this.getTaskGitCloneRepository(),
+                this.getTaskGitCherryPick(),
+                new ScriptTask()
+                    .description("Run php lint")
+                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                    .inlineBody(
+                        this.getScriptTaskBashInlineBody() +
+                        "find . -name \\*.php -print0 | xargs -0 -n1 -P2 php -l >/dev/null\n"
+                    )
+            )
+            .requirements(
+                requirement
+            );
+    }
+
+    /**
+     * Job for unit testing PHP
+     *
+     * @param Requirement requirement
+     * @param String requirementIdentfier
+     */
+    protected Job getJobUnitPhp(Requirement requirement, String requirementIdentifier) {
+        return new Job("Unit " + requirementIdentifier, new BambooKey("UT" + requirementIdentifier))
+            .description("Run unit tests " + requirementIdentifier)
+            .tasks(
+                this.getTaskGitCloneRepository(),
+                this.getTaskGitCherryPick(),
+                this.getTaskComposerInstall(),
+                new ScriptTask()
+                    .description("Run phpunit")
+                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                    .inlineBody(
+                        this.getScriptTaskBashInlineBody() +
+                        this.getScriptTaskBashPhpNoXdebug() +
+                        "php_no_xdebug bin/phpunit --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "UnitTests.xml"
+                    )
+            )
+            .finalTasks(
+                new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
+                    .resultDirectories("test-reports/phpunit.xml")
+            )
+            .requirements(
+                requirement
+            );
+    }
+
+    /**
+     * Task definition for basic core clone of linked default repository
+     */
+    protected Task getTaskGitCloneRepository() {
+        return new VcsCheckoutTask()
+            .description("Checkout git core")
+            .checkoutItems(new CheckoutItem().defaultRepository())
+            .cleanCheckout(true);
+    }
+
+    /**
+     * Task definition to cherry pick a patch set from gerrit on top of cloned core
+     */
+    protected Task getTaskGitCherryPick() {
+        return new ScriptTask()
+            .description("Gerrit cherry pick")
+            .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+            .inlineBody(
+                this.getScriptTaskBashInlineBody() +
+                "CHANGEURL=${bamboo.changeUrl}\n" +
+                "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
+                "PATCHSET=${bamboo.patchset}\n" +
+                "\n" +
+                "if [[ $CHANGEURL ]]; then\n" +
+                "    gerrit-cherry-pick https://review.typo3.org/Packages/TYPO3.CMS $CHANGEURLID/$PATCHSET || exit 1\n" +
+                "fi\n"
+            );
+    }
+
+    /**
+     * Task definition to execute composer install
+     */
+    protected Task getTaskComposerInstall() {
+        return new CommandTask()
+            .description("composer install")
+            .executable("composer")
+            .argument("install -n");
+    }
+
+    /**
+     * Task to delete any created mysql test databases, used as final task
+     */
+    protected Task getTaskDeleteMysqlDatabases() {
+        return new ScriptTask()
+            .description("Delete mysql test dbs")
+            .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+            .inlineBody(
+                this.getScriptTaskBashInlineBody() +
+                "DB_STARTS_WITH=\"func_\"\n" +
+                "MUSER=\"funcu\"\n" +
+                "MPWD=\"funcp\"\n" +
+                "MYSQL=\"mysql\"\n" +
+                "DBS=\"$($MYSQL -u $MUSER -p\"$MPWD\" -Bse 'show databases')\"\n" +
+                "\n" +
+                "for db in $DBS; do\n" +
+                "    if [[ \"$db\" == $DB_STARTS_WITH* ]]; then\n" +
+                "        echo \"Deleting $db\"\n" +
+                "        $MYSQL -u $MUSER -p\"$MPWD\" -Bse \"drop database $db\"\n" +
+                "    fi\n" +
+                "done\n"
+            );
+    }
+
+    /**
+     * Task to split functional jobs into chunks
+     */
+    protected Task getTaskSplitFunctionalJobs(int numberOfJobs) {
+        return new ScriptTask()
+            .description("Create list of test files to execute per job")
+            .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+            .inlineBody(
+                this.getScriptTaskBashInlineBody() +
+                "./" + this.testingFrameworkBuildPath + "Scripts/splitFunctionalTests.sh " + numberOfJobs
+            );
+    }
+
+    /**
+     * Requirement for php 5.5
+     */
+    protected Requirement getRequirementPhpVersion55() {
+        return new Requirement("system.phpVersion")
+            .matchValue("5.5")
+            .matchType(Requirement.MatchType.EQUALS);
+    }
+
+    /**
+     * Requirement for php 5.6
+     */
+    protected Requirement getRequirementPhpVersion56() {
+        return new Requirement("system.phpVersion")
+            .matchValue("5.6")
+            .matchType(Requirement.MatchType.EQUALS);
+    }
+
+    /**
+     * Requirement for php 7.0
+     */
+    protected Requirement getRequirementPhpVersion70() {
+        return new Requirement("system.phpVersion")
+            .matchValue("7.0")
+            .matchType(Requirement.MatchType.EQUALS);
+    }
+
+    /**
+     * Requirement for php 7.1
+     */
+    protected Requirement getRequirementPhpVersion71() {
+        return new Requirement("system.phpVersion")
+            .matchValue("7.1")
+            .matchType(Requirement.MatchType.EQUALS);
+    }
+
+    /**
+     * A bash header for script tasks forking a bash if needed
+     */
+    protected String getScriptTaskBashInlineBody() {
+        return
+            "#!/bin/bash\n" +
+            "\n" +
+            "if [ \"$(ps -p \"$$\" -o comm=)\" != \"bash\" ]; then\n" +
+            "    bash \"$0\" \"$@\"\n" +
+            "    exit \"$?\"\n" +
+            "fi\n" +
+            "\n";
+    }
+
+    /**
+     * A bash function providing a php bin without xdebug
+     */
+    protected String getScriptTaskBashPhpNoXdebug() {
+        return
+            "php_no_xdebug () {\n" +
+            "    temporaryPath=\"$(mktemp -t php.XXXX).ini\"\n" +
+            "    php -i | grep \"\\.ini\" | grep -o -e '\\(/[A-Za-z0-9._-]\\+\\)\\+\\.ini' | grep -v xdebug | xargs awk 'FNR==1{print \"\"}1' > \"${temporaryPath}\"\n" +
+            "    php -n -c \"${temporaryPath}\" \"$@\"\n" +
+            "    RETURN=$?\n" +
+            "    rm -f \"${temporaryPath}\"\n" +
+            "    exit $RETURN\n" +
+            "}\n" +
+            "\n";
+    }
+}
diff --git a/Build/bamboo/src/main/java/core/PreMergeSpec.java b/Build/bamboo/src/main/java/core/PreMergeSpec.java
new file mode 100644 (file)
index 0000000..31ac58a
--- /dev/null
@@ -0,0 +1,135 @@
+package core;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+import java.util.ArrayList;
+
+import com.atlassian.bamboo.specs.api.BambooSpec;
+import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.api.builders.plan.Job;
+import com.atlassian.bamboo.specs.api.builders.plan.Plan;
+import com.atlassian.bamboo.specs.api.builders.plan.Stage;
+import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
+import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
+import com.atlassian.bamboo.specs.api.builders.project.Project;
+import com.atlassian.bamboo.specs.builders.task.ScriptTask;
+import com.atlassian.bamboo.specs.builders.trigger.RemoteTrigger;
+import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
+import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
+import com.atlassian.bamboo.specs.util.BambooServer;
+
+/**
+ * Core 7.6 pre-merge test plan.
+ */
+@BambooSpec
+public class PreMergeSpec extends AbstractCoreSpec {
+
+    protected int numberOfFunctionalMysqlJobs = 5;
+
+    /**
+     * Run main to publish plan on Bamboo
+     */
+    public static void main(final String[] args) throws Exception {
+        // By default credentials are read from the '.credentials' file.
+        BambooServer bambooServer = new BambooServer("https://bamboo.typo3.com:443");
+        Plan plan = new PreMergeSpec().createPlan();
+        bambooServer.publish(plan);
+    }
+
+    /**
+     * Core 7.6 pre-merge plan is in "TYPO3 core" project of bamboo
+     */
+    Project project() {
+        return new Project().name("TYPO3 Core").key("CORE");
+    }
+
+    /**
+     * Returns full Plan definition
+     */
+    Plan createPlan() {
+        // PREPARATION stage
+        ArrayList<Job> jobsPreparationStage = new ArrayList<Job>();
+
+        jobsPreparationStage.add(this.getJobBuildLabels());
+
+        jobsPreparationStage.add(this.getJobComposerValidate());
+
+        Stage stagePreparation = new Stage("Preparation")
+            .jobs(jobsPreparationStage.toArray(new Job[jobsPreparationStage.size()]));
+
+
+        // MAIN stage
+        ArrayList<Job> jobsMainStage = new ArrayList<Job>();
+
+        jobsMainStage.add(this.getJobIntegrationVarious());
+
+        jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, this.getRequirementPhpVersion55(), "PHP55"));
+        jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, this.getRequirementPhpVersion56(), "PHP56"));
+        jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, this.getRequirementPhpVersion70(), "PHP70"));
+        jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, this.getRequirementPhpVersion71(), "PHP71"));
+
+        jobsMainStage.add(this.getJobLintPhp(this.getRequirementPhpVersion55(), "PHP55"));
+        jobsMainStage.add(this.getJobLintPhp(this.getRequirementPhpVersion56(), "PHP56"));
+        jobsMainStage.add(this.getJobLintPhp(this.getRequirementPhpVersion70(), "PHP70"));
+        jobsMainStage.add(this.getJobLintPhp(this.getRequirementPhpVersion71(), "PHP71"));
+
+        jobsMainStage.add(this.getJobUnitPhp(this.getRequirementPhpVersion55(), "PHP55"));
+        jobsMainStage.add(this.getJobUnitPhp(this.getRequirementPhpVersion56(), "PHP56"));
+        jobsMainStage.add(this.getJobUnitPhp(this.getRequirementPhpVersion70(), "PHP70"));
+        jobsMainStage.add(this.getJobUnitPhp(this.getRequirementPhpVersion71(), "PHP71"));
+
+        Stage stageMainStage = new Stage("Main stage")
+            .jobs(jobsMainStage.toArray(new Job[jobsMainStage.size()]));
+
+
+        // Compile plan
+        return new Plan(project(), "Core 7.6 pre-merge", "GTC76")
+            .description("Execute TYPO3 core 7.6 pre-merge tests. Auto generated! See Build/bamboo of core git repository.")
+            .stages(
+                stagePreparation,
+                stageMainStage
+            )
+            .linkedRepositories("git.typo3.org Core 7.6")
+            .triggers(
+                new RepositoryPollingTrigger()
+                    .name("Repository polling for post-merge builds"),
+                new RemoteTrigger()
+                    .name("Remote trigger for pre-merge builds")
+                    .description("Gerrit")
+                    .triggerIPAddresses("5.10.165.218,91.184.35.13"))
+            .variables(
+                new Variable("changeUrl", ""),
+                new Variable("patchset", "")
+            )
+            .planBranchManagement(
+                new PlanBranchManagement()
+                    .delete(new BranchCleanup())
+                    .notificationForCommitters()
+            );
+    }
+
+    /**
+     * Job creating labels needed for intercept communication
+     */
+    protected Job getJobBuildLabels() {
+        return new Job("Create build labels", new BambooKey("CLFB"))
+            .description("Create changeId and patch set labels from variable access and parsing result of a dummy task")
+            .tasks(
+                new ScriptTask()
+                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                    .inlineBody("echo \"I'm just here for the labels!\"")
+            );
+    }
+}
diff --git a/Build/bamboo/src/test/java/core/PreMergeSpecTest.java b/Build/bamboo/src/test/java/core/PreMergeSpecTest.java
new file mode 100644 (file)
index 0000000..6de2cc6
--- /dev/null
@@ -0,0 +1,29 @@
+package core;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+import org.junit.Test;
+
+import com.atlassian.bamboo.specs.api.builders.plan.Plan;
+import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
+import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
+
+public class PreMergeSpecTest {
+    @Test
+    public void checkYourPlanOffline() throws PropertiesValidationException {
+        Plan plan = new PreMergeSpec().createPlan();
+
+        EntityPropertiesBuilders.build(plan);
+    }
+}