[TASK] Enable functional tests on microsoft sql server again 41/57641/10
authorChristian Kuhn <lolli@schwarzbu.ch>
Fri, 20 Jul 2018 12:11:26 +0000 (14:11 +0200)
committerChristian Kuhn <lolli@schwarzbu.ch>
Sun, 22 Jul 2018 14:24:37 +0000 (16:24 +0200)
Executing functional tests on mssql has been disabled a while
ago since the daemon crashed frequently: The server can not
run on tmpfs volumes that we usually use to speed up database
operations. sql server uses O_DIRECT i/o system calls which
tmpfs does not support. Find details at
https://github.com/Microsoft/mssql-docker/issues/12

We now run mssql tests on hard disks again, but enable them
only as nightly since they are too slow as pre-merge tests.

Change-Id: Iefc61670dcf74012fcdaada8e5b771538e7cf8c3
Resolves: #85614
Releases: master
Reviewed-on: https://review.typo3.org/57641
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
Build/bamboo/src/main/java/core/AbstractCoreSpec.java
Build/bamboo/src/main/java/core/NightlySpec.java
Build/bamboo/src/main/java/core/PreMergeSpec.java
Build/testing-docker/bamboo/docker-compose.yml
typo3/sysext/core/Classes/Database/Connection.php
typo3/sysext/core/Tests/Functional/Error/ErrorHandlerTest.php

index 305e2a7..76a4a1d 100644 (file)
@@ -50,18 +50,6 @@ abstract public class AbstractCoreSpec {
     protected String testingFrameworkBuildPath = "vendor/typo3/testing-framework/Resources/Core/Build/";
 
     /**
-     * @todo This can be removed if acceptance mssql functional tests work again
-     */
-    protected String credentialsMssql =
-        "typo3DatabaseDriver=\"sqlsrv\"" +
-        " typo3DatabaseName=\"func\"" +
-        " typo3DatabasePassword='Test1234!'" +
-        " typo3DatabaseUsername=\"SA\"" +
-        " typo3DatabaseHost=\"localhost\"" +
-        " typo3DatabasePort=\"1433\"" +
-        " typo3DatabaseCharset=\"utf-8\"";
-
-    /**
      * Default permissions on core plans
      *
      * @param projectName
@@ -133,6 +121,45 @@ abstract public class AbstractCoreSpec {
     }
 
     /**
+     * Job creating labels needed for intercept communication
+     */
+    protected Job getJobBuildLabels() {
+        return new Job("Create build labels", new BambooKey("CLFB"))
+            .description("Create changeId and patch set labels from variable access and parsing result of a dummy task")
+            .pluginConfigurations(new AllOtherPluginsConfiguration()
+                .configuration(new MapBuilder()
+                    .put("repositoryDefiningWorkingDirectory", -1)
+                    .put("custom", new MapBuilder()
+                        .put("auto", new MapBuilder()
+                            .put("regex", "https:\\/\\/review\\.typo3\\.org\\/(#\\/c\\/)?(\\d+)")
+                            .put("label", "change-\\2\\, patchset-${bamboo.patchset}")
+                            .build()
+                        )
+                        .put("buildHangingConfig.enabled", "false")
+                        .put("ncover.path", "")
+                        .put("clover", new MapBuilder()
+                            .put("path", "")
+                            .put("license", "")
+                            .put("useLocalLicenseKey", "true")
+                            .build()
+                        )
+                        .build()
+                    )
+                    .build()
+                )
+            )
+            .tasks(
+                new ScriptTask()
+                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+                    .inlineBody("echo \"I'm just here for the labels!\"")
+            )
+            .requirements(
+                this.getRequirementDocker10()
+            )
+            .cleanWorkingDirectory(true);
+    }
+
+    /**
      * Job composer validate
      *
      * @param String requirementIdentifier
@@ -462,8 +489,6 @@ abstract public class AbstractCoreSpec {
     /**
      * Jobs for mssql based functional tests
      *
-     * @todo Currently disabled and broken
-     *
      * @param int numberOfChunks
      * @param String requirementIdentifier
      */
@@ -482,15 +507,37 @@ abstract public class AbstractCoreSpec {
                     this.getTaskGitCloneRepository(),
                     this.getTaskGitCherryPick(),
                     this.getTaskComposerInstall(requirementIdentifier),
+                    this.getTaskDockerDependenciesFunctionalMssql(),
                     this.getTaskSplitFunctionalJobs(numberOfChunks, requirementIdentifier),
                     new ScriptTask()
                         .description("Run phpunit with functional chunk " + formattedI)
                         .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
                         .inlineBody(
                             this.getScriptTaskBashInlineBody() +
-                            "./bin/phpunit --exclude-group not-mssql --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
+                            "function phpunit() {\n" +
+                            "    docker run \\\n" +
+                            "        -u ${HOST_UID} \\\n" +
+                            "        -v /bamboo-data/${BAMBOO_COMPOSE_PROJECT_NAME}/passwd:/etc/passwd \\\n" +
+                            "        -v ${BAMBOO_COMPOSE_PROJECT_NAME}_bamboo-data:/srv/bamboo/xml-data/build-dir/ \\\n" +
+                            "        -e typo3DatabaseDriver=sqlsrv \\\n" +
+                            "        -e typo3DatabaseName=func \\\n" +
+                            "        -e typo3DatabasePassword=Test1234! \\\n" +
+                            "        -e typo3DatabaseUsername=SA \\\n" +
+                            "        -e typo3DatabaseHost=localhost \\\n" +
+                            "        -e typo3DatabasePort=1433 \\\n" +
+                            "        -e typo3DatabaseCharset=utf-8 \\\n" +
+                            "        -e typo3DatabaseHost=mssql2017cu9 \\\n" +
+                            "        -e typo3TestingRedisHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_redis4_1 \\\n" +
+                            "        -e typo3TestingMemcachedHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_memcached1-5_1 \\\n" +
+                            "        --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
+                            "        --network ${BAMBOO_COMPOSE_PROJECT_NAME}_test \\\n" +
+                            "        --rm \\\n" +
+                            "        typo3gmbh/" + requirementIdentifier.toLowerCase() + ":latest \\\n" +
+                            "        bin/bash -c \"cd ${PWD}; ./bin/phpunit $*\"\n" +
+                            "}\n" +
+                            "\n" +
+                            "phpunit --exclude-group not-mssql --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
                         )
-                        .environmentVariables(this.credentialsMssql)
                 )
                 .finalTasks(
                     this.getTaskStopDockerDependencies(),
@@ -501,7 +548,6 @@ abstract public class AbstractCoreSpec {
                     this.getRequirementDocker10()
                 )
                 .cleanWorkingDirectory(true)
-                .enabled(false)
             );
         }
 
@@ -1264,6 +1310,21 @@ abstract public class AbstractCoreSpec {
     }
 
     /**
+     * Start docker sibling containers to execute functional tests on mariadb
+     */
+    protected Task getTaskDockerDependenciesFunctionalMssql() {
+        return new ScriptTask()
+            .description("Start docker siblings for functional tests on mssql")
+            .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+            .inlineBody(
+                this.getScriptTaskBashInlineBody() +
+                "cd Build/testing-docker/bamboo\n" +
+                "echo COMPOSE_PROJECT_NAME=${BAMBOO_COMPOSE_PROJECT_NAME}sib > .env\n" +
+                "docker-compose run start_dependencies_functional_mssql"
+            );
+    }
+
+    /**
      * Start docker sibling containers to execute functional tests on postgres
      */
     protected Task getTaskDockerDependenciesFunctionalPostgres10() {
index b67a67d..3836137 100644 (file)
@@ -40,9 +40,9 @@ public class NightlySpec extends AbstractCoreSpec {
 
     protected int numberOfAcceptanceTestJobs = 8;
     protected int numberOfFunctionalMysqlJobs = 6;
-    protected int numberOfFunctionalMssqlJobs = 6;
+    protected int numberOfFunctionalMssqlJobs = 16;
     protected int numberOfFunctionalPgsqlJobs = 6;
-    protected int numberOfFunctionalSqliteJobs = 10;
+    protected int numberOfFunctionalSqliteJobs = 6;
     protected int numberOfUnitRandomOrderJobs = 4;
 
     /**
@@ -66,6 +66,12 @@ public class NightlySpec extends AbstractCoreSpec {
      * Returns full Plan definition
      */
     Plan createPlan() {
+        // PREPARATION stage
+        ArrayList<Job> jobsPreparationStage = new ArrayList<Job>();
+        jobsPreparationStage.add(this.getJobBuildLabels());
+        Stage stagePreparation = new Stage("Preparation")
+            .jobs(jobsPreparationStage.toArray(new Job[jobsPreparationStage.size()]));
+
         // MAIN stage
         ArrayList<Job> jobsMainStage = new ArrayList<Job>();
 
@@ -107,6 +113,7 @@ public class NightlySpec extends AbstractCoreSpec {
             .description("Execute TYPO3 core master nightly tests. Auto generated! See Build/bamboo of core git repository.")
             .pluginConfigurations(this.getDefaultPlanPluginConfiguration())
             .stages(
+                stagePreparation,
                 stageMainStage
             )
             .linkedRepositories("github TYPO3 TYPO3.CMS")
index 9f01fb7..eb507a9 100644 (file)
@@ -26,7 +26,6 @@ import com.atlassian.bamboo.specs.api.builders.plan.Plan;
 import com.atlassian.bamboo.specs.api.builders.plan.Stage;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
-import com.atlassian.bamboo.specs.api.builders.plan.configuration.AllOtherPluginsConfiguration;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.api.builders.requirement.Requirement;
 import com.atlassian.bamboo.specs.builders.notification.PlanCompletedNotification;
@@ -35,7 +34,6 @@ import com.atlassian.bamboo.specs.builders.trigger.RemoteTrigger;
 import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
 import com.atlassian.bamboo.specs.util.BambooServer;
-import com.atlassian.bamboo.specs.util.MapBuilder;
 
 /**
  * Core master pre-merge test plan.
@@ -101,7 +99,8 @@ public class PreMergeSpec extends AbstractCoreSpec {
         jobsMainStage.add(this.getJobIntegrationVarious("PHP72"));
 
         jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, "PHP72"));
-        jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, "PHP72"));
+        // mssql functionals are not executed as pre-merge
+        // jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, "PHP72"));
         jobsMainStage.addAll(this.getJobsFunctionalTestsPgsql(this.numberOfFunctionalPgsqlJobs, "PHP72"));
         jobsMainStage.addAll(this.getJobsFunctionalTestsSqlite(this.numberOfFunctionalSqliteJobs, "PHP72"));
 
@@ -153,45 +152,6 @@ public class PreMergeSpec extends AbstractCoreSpec {
     }
 
     /**
-     * Job creating labels needed for intercept communication
-     */
-    protected Job getJobBuildLabels() {
-        return new Job("Create build labels", new BambooKey("CLFB"))
-            .description("Create changeId and patch set labels from variable access and parsing result of a dummy task")
-            .pluginConfigurations(new AllOtherPluginsConfiguration()
-                .configuration(new MapBuilder()
-                    .put("repositoryDefiningWorkingDirectory", -1)
-                    .put("custom", new MapBuilder()
-                        .put("auto", new MapBuilder()
-                            .put("regex", "https:\\/\\/review\\.typo3\\.org\\/(#\\/c\\/)?(\\d+)")
-                            .put("label", "change-\\2\\, patchset-${bamboo.patchset}")
-                            .build()
-                        )
-                        .put("buildHangingConfig.enabled", "false")
-                        .put("ncover.path", "")
-                        .put("clover", new MapBuilder()
-                            .put("path", "")
-                            .put("license", "")
-                            .put("useLocalLicenseKey", "true")
-                            .build()
-                        )
-                        .build()
-                    )
-                    .build()
-                )
-            )
-            .tasks(
-                new ScriptTask()
-                    .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
-                    .inlineBody("echo \"I'm just here for the labels!\"")
-            )
-            .requirements(
-                this.getRequirementDocker10()
-            )
-            .cleanWorkingDirectory(true);
-    }
-
-    /**
      * Job checking CGL of last git commit
      *
      * @param String requirementIdentifier
index 346bf71..b54052a 100644 (file)
@@ -23,6 +23,16 @@ services:
       - /var/lib/postgresql/data:rw,noexec,nosuid
     networks:
       - test
+  mssql2017cu9:
+    image: microsoft/mssql-server-linux:2017-CU9
+    environment:
+      ACCEPT_EULA: Y
+      SA_PASSWORD: Test1234!
+      MSSQL_PID: Developer
+    volumes:
+      - /etc/passwd:/etc/passwd,ro
+    networks:
+      - test
   redis4:
     image: redis:4-alpine
     networks:
@@ -123,6 +133,24 @@ services:
         echo Connected!;
       "
 
+  start_dependencies_functional_mssql:
+    image: alpine:3.8
+    links:
+      - mssql2017cu9
+      - redis4
+      - memcached1-5
+    networks:
+      - test
+    command: >
+      /bin/sh -c "
+        echo Waiting for db start...;
+        while ! nc -z mssql2017cu9 1433;
+        do
+          sleep 1;
+        done;
+        echo Connected!;
+      "
+
   start_dependencies_functional_postgres10:
     image: alpine:3.8
     links:
index f39d768..2c239f9 100644 (file)
@@ -20,6 +20,7 @@ use Doctrine\DBAL\Configuration;
 use Doctrine\DBAL\Driver;
 use Doctrine\DBAL\Driver\Statement;
 use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
+use Doctrine\DBAL\Platforms\SQLServer2012Platform;
 use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerAwareTrait;
 use TYPO3\CMS\Core\Database\Query\Expression\ExpressionBuilder;
@@ -448,9 +449,16 @@ class Connection extends \Doctrine\DBAL\Connection implements LoggerAwareInterfa
      */
     public function lastInsertId($tableName = null, string $fieldName = 'uid'): string
     {
-        if ($this->getDatabasePlatform() instanceof PostgreSqlPlatform) {
+        $databasePlatform = $this->getDatabasePlatform();
+        if ($databasePlatform instanceof PostgreSqlPlatform) {
             return parent::lastInsertId(trim(implode('_', [$tableName, $fieldName, 'seq']), '_'));
         }
+        if ($databasePlatform instanceof SQLServer2012Platform) {
+            // lastInsertId() in mssql >2012 takes a sequence name and not the table name as
+            // argument. If no argument is given, last insert id of latest table is returned.
+            // https://docs.microsoft.com/de-de/sql/connect/php/pdo-lastinsertid?view=sql-server-2017
+            return (string)parent::lastInsertId();
+        }
 
         return (string)parent::lastInsertId($tableName);
     }
index 1833e47..f7d3131 100644 (file)
@@ -36,11 +36,12 @@ class ErrorHandlerTest extends FunctionalTestCase
     ];
 
     /**
-     * Disabled on sqlite: It does not support init command "SET NAMES 'UTF8'". That's ok
-     * since this test is not about db platform support but error handling in core.
+     * Disabled on sqlite and mssql: They don't support init command "SET NAMES 'UTF8'". That's
+     * ok since this test is not about db platform support but error handling in core.
      *
      * @test
      * @group not-sqlite
+     * @group not-mssql
      */
     public function handleErrorFetchesDeprecations()
     {