Merge remote-tracking branch 'origin/develop' into...

Merge remote-tracking branch 'origin/develop' into 415-send-information-mail-when-a-extension-key-was-transferred

# Conflicts:
#	extensions/ter_fe2/Classes/Controller/RegisterkeyController.php
#	extensions/ter_fe2/composer.json

.ddev/.gitignore

 #ddev-generated: Automatically generated ddev .gitignore.
 # You can remove the above line if you want to edit and maintain this file yourself.
-
-/commands/*/*.example
-/commands/*/README.txt
-/commands/host/launch
-/commands/web/xdebug
-/commands/db/mysql
-/homeadditions/*.example
-/homeadditions/README.txt
 /.gitignore
-/import.yaml
+
+/**/*.example
+/.dbimageBuild
+/.dbimageExtra
 /.ddev-docker-compose-base.yaml
 /.ddev-docker-compose-full.yaml
-/db_snapshots
-/sequelpro.spf
-/import-db
-/config.*.y*ml
-/.webimageBuild
-/.dbimageBuild
+/.ddevlive-downloads
+/.global_commands
+/.homeadditions
 /.sshimageBuild
+/.webimageBuild
 /.webimageExtra
-/.dbimageExtra
-/*-build/Dockerfile.example
+/apache/apache-site.conf
+/commands/.gitattributes
+/commands/db/mysql
+/commands/host/launch
+/commands/web/live
+/commands/web/xdebug
+/config.*.y*ml
+/db_snapshots
+/import-db
+/import.yaml
+/nginx_full/nginx-site.conf
+/sequelpro.spf
+/**/README.*

.ddev/config.yaml

 name: ter
 type: typo3
 docroot: public
-php_version: "7.2"
+php_version: "7.4"
 webserver_type: nginx-fpm
 router_http_port: "80"
 router_https_port: "443"
 xdebug_enabled: false
 additional_hostnames: []
 additional_fqdns: []
-mariadb_version: "10.2"
+mariadb_version: "10.3"
 provider: default
 hooks:
   post-import-db:
   - exec: vendor/bin/typo3cms database:updateschema
-  - exec: vendor/bin/typo3cms backend:createadmin --username=admin --password=password
+  - exec: vendor/bin/typo3cms backend:createadmin admin password
   post-start:
   - exec: cp private/typo3conf/AdditionalConfiguration.ddev.php private/typo3conf/AdditionalConfiguration.php
   - exec: composer install
   - exec: vendor/bin/typo3cms install:generatepackagestates
   - exec: vendor/bin/typo3cms install:extensionsetupifpossible
 omit_containers: [dba]
-webimage_extra_packages: [php7.2-ldap]
+webimage_extra_packages: [php7.4-ldap]
 use_dns_when_possible: true
 timezone: Europe/Berlin
 
@@ -158,9 +158,9 @@ timezone: Europe/Berlin
 # In this case the user must provide all such settings.
 
 # no_project_mount: false
-# (Experimental) If true, ddev will not mount the project into the web container; 
+# (Experimental) If true, ddev will not mount the project into the web container;
 # the user is responsible for mounting it manually or via a script.
-# This is to enable experimentation with alternate file mounting strategies. 
+# This is to enable experimentation with alternate file mounting strategies.
 # For advanced users only!
 
 # provider: default # Currently either "default" or "pantheon"

.ddev/docker-compose.chromedriver.yaml

+version: '3.6'
+
+services:
+
+  chromedriver:
+    container_name: ddev-${DDEV_SITENAME}-chromedriver
+    image: robcherry/docker-chromedriver
+    restart: "no"
+    ports:
+      - "4444"
+    environment:
+      - CHROMEDRIVER_WHITELISTED_IPS=""
+    external_links:
+      - "ddev-router:${DDEV_SITENAME}.ddev.site"

.gitignore

@@ -13,3 +13,5 @@ sequelpro.spf
 .php_cs.cache
 .ddev/db_snapshots/
 .idea/
+/Tests/_output/
+/Tests/_support/_generated/

.gitlab-ci.yml

 include:
   - project: 't3o/t3olayout'
-    ref: master
+    ref: main
     file: '/Configuration/GitLab/t3o-builds.yml'
 
 stages:
@@ -23,32 +23,24 @@ variables:
 
 test:unit:
   stage: test
-  image: composer:1
+  image: php:7.4
   variables:
     GIT_STRATEGY: "clone"
-    GIT_SUBMODULE_STRATEGY: "recursive"
   before_script:
-    - apk add git --update
+    - set -xe
+    - apt-get update -yqq
+    - apt-get install git unzip zlib1g-dev libzip-dev -yqq
+    - docker-php-ext-install zip
+    - pecl install xdebug
+    - docker-php-ext-enable xdebug
+    - curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
     - export TYPO3_PATH_WEB="$PWD/private"
     - composer config cache-dir /cache/composer
     - mkdir $TYPO3_PATH_WEB/fileadmin/ && touch $TYPO3_PATH_WEB/fileadmin/currentcoredata.json
   script:
     - composer install --ignore-platform-reqs
     - composer test:unit
-
-layout:
-  stage: layout
-  image: node:8
-  before_script:
-    - cd private/typo3conf/ext/t3olayout/Build
-  script:
-    - npm install
-    - npm run build
-  after_script:
-    - rm -Rf private/typo3conf/ext/t3olayout/Build/node_modules
   artifacts:
-    paths:
-      - ./
-    expire_in: '4h'
-  dependencies:
-    - build
+    expire_in: 15 mins
+    reports:
+      junit: build/*-report.xml

.gitlab-ci/Tests/UnitTests.xml

@@ -2,7 +2,7 @@
         backupGlobals="true"
         backupStaticAttributes="false"
         bootstrap="../../vendor/nimut/testing-framework/res/Configuration/UnitTestsBootstrap.php"
-        colors="true"
+        colors="false"
         convertErrorsToExceptions="true"
         convertWarningsToExceptions="true"
         forceCoversAnnotation="false"
@@ -18,4 +18,13 @@
             <directory>../../extensions/ter_fe2/Tests/Unit</directory>
         </testsuite>
     </testsuites>
+    <filter>
+        <whitelist processUncoveredFilesFromWhitelist="true">
+            <directory suffix=".php">../../extensions/ter_fe2/Classes/</directory>
+        </whitelist>
+    </filter>
+    <logging>
+        <log type="coverage-text" target="php://stdout" showUncoveredFiles="false"/>
+        <log type="junit" target="../../build/junit-report.xml"/>
+    </logging>
 </phpunit>

.gitlab-ci/deployer/deploy.php

@@ -8,7 +8,6 @@ require_once '/tmp/vendor/deployer/recipes/recipe/rsync.php';
 $sharedDirectories = [
     'private/fileadmin',
     'private/uploads',
-    'public/.well-known',
     'var'
 ];
 set('shared_dirs', $sharedDirectories);
@@ -59,6 +58,8 @@ set('bin_folder', 'vendor/bin/');
 task('typo3', function() {
     run('cd {{release_path}} && {{php}} {{bin_folder}}typo3cms install:generatepackagestates');
     run('cd {{release_path}} && {{php}} {{bin_folder}}typo3cms install:extensionsetupifpossible');
+    run('cd {{release_path}} && {{php}} {{bin_folder}}typo3cms upgrade:prepare -q');
+    run('cd {{release_path}} && {{php}} {{bin_folder}}typo3cms upgrade:run databaseRowsUpdateWizard pagesLanguageOverlayBeGroupsAccessRights');
 });
 
 task('solr:host', function() {

scripts @ 57323530

-Subproject commit 57323530b0771c5575efb38d64d1bf117716ebf5

.gitmodules

-[submodule ".gitlab-ci"]
-	path = .gitlab-ci/scripts
-	url = https://github.com/torvitas/docker-scripts.git

README.md

+# extensions.typo3.org documentation
+
+## Acceptance Tests
+
+### Concepts
+
+The acceptance tests run in their own environment. In this environment we import a basic database via codeception
+to have a working site and add new data per test via yaml files and steps provided by the 
+`punktde/codeception-database` package. When a yaml file is imported via the gherkin step, every database table 
+found in the yaml file is truncated before the new data is imported.
+
+In `Tests/acceptance.suite.yml` the config for the database module can be found. We have set 
+`populate` and `reset` to true, so before running any test and between every test the command given as the config
+option `populator` is run. If no `populator` is given, the `dumpFile` is imported. 
+
+At the moment (Nov 4th 2019) GeckoDriver and FireFox testing is only possible by using the detour via selenium,
+and even then it's not really reliable.
+
+We are running chromedriver within ddev, so if you use ddev everything just works ;)
+
+### Run codeception tests
+
+#### Run the tests
+
+```
+ddev run --service web vendor/bin/codecept --steps run acceptance 
+```
+
+With `--steps`, all run steps are outputted to the command line. All errors are reported even if you don't use the 
+flag. You can also add `--xml` as a flag to get JUnit compatible output.
+
+If you get errors, you can find the output in the directory `Tests/_output`. In there is a list of failed tests and 
+a screenshot and the html output of the page the moment a test failed. 
+
+##### Single feature
+
+To run a single feature just add the path to the feature file to your command, e.g.
+ 
+```
+ddev run --service web vendor/bin/codecept --steps run acceptance Tests/acceptance/Features/Login.feature
+```
+
+You can also run single scenarios by providing step names, e.g.
+
+```
+ddev run --service web vendor/bin/codecept --steps run acceptance Tests/acceptance/Features/Login.feature:Login\ and\ Logout
+```
+
+### Writing tests
+
+The acceptance tests are put into the directory `Tests/acceptance/Feature`. You can use any folder structure to 
+group your tests.
+To get all the available step definitions just run the command
+
+```
+ddev run --service web vendor/bin/codecept gherkin:steps acceptance
+```
+
+which outputs the steps added to the project.
+
+To add new steps, just write the new step in your `.feature` file and run 
+
+```
+ddev run --service web vendor/bin/codecept gherkin:snippets acceptance
+```
+
+This will output a function stub with the proper step definition phpdoc. Put this in a appropriate `ActorTrait` 
+and (if needed) add the `ActorTrait` to the `AcceptanceTester` (`Tests/_support/AcceptanceTester.php`). Example
+traits can be found in the directory `Classes/ActorTraits/` of the `punktde/codeception-*` packages.

Tests/_support/AcceptanceTester.php

+<?php
+
+use PunktDe\Codeception\Database\ActorTraits\Database;
+use PunktDe\Codeception\Webdriver\ActorTraits\Webdriver;
+
+/**
+ * Inherited Methods
+ * @method void wantToTest($text)
+ * @method void wantTo($text)
+ * @method void execute($callable)
+ * @method void expectTo($prediction)
+ * @method void expect($prediction)
+ * @method void amGoingTo($argumentation)
+ * @method void am($role)
+ * @method void lookForwardTo($achieveValue)
+ * @method void comment($description)
+ * @method void pause()
+ *
+ * @SuppressWarnings(PHPMD)
+*/
+class AcceptanceTester extends \Codeception\Actor
+{
+    use _generated\AcceptanceTesterActions;
+
+    use Webdriver;
+    use Database;
+}

Tests/_support/Helper/Acceptance.php

+<?php
+
+namespace Helper;
+
+// here you can define custom actions
+// all public methods declared in helper class will be available in $I
+
+use Codeception\Configuration;
+use Codeception\Exception\ConfigurationException;
+use Codeception\Exception\ModuleException;
+use Codeception\Module\Cli;
+use Codeception\Module\Filesystem;
+use Neos\Utility\Files;
+use PunktDe\Codeception\Database\Module\Database;
+
+class Acceptance extends \Codeception\Module
+{
+
+    /**
+     * @param string[] $settings
+     * @throws ConfigurationException
+     * @throws ModuleException
+     */
+    public function _beforeSuite($settings = []): void
+    {
+        $filesystem = $this->getModule('Filesystem');
+        /** @var Filesystem $filesystem */
+        $filesystem->cleanDir(Configuration::outputDir());
+
+        $cli = $this->getModule('Cli');
+        /** @var Cli $cli */
+        $db = $this->getModule(Database::class);
+        /** @var Database $db */
+        $this->debug('dumping development database');
+
+        $dumpCommand = sprintf(
+            'mysqldump --host=%s --user=%s --password=%s %s > %s',
+            $db->_getConfig('user'),
+            $db->_getConfig('user'),
+            $db->_getConfig('user'),
+            $db->_getConfig('user'),
+            Files::concatenatePaths([Configuration::dataDir(), 'localDatabase.sql'])
+        );
+
+        $cli->runShellCommand($dumpCommand);
+    }
+
+    public function _afterSuite()
+    {
+        $cli = $this->getModule('Cli');
+        /** @var Cli $cli */
+        $db = $this->getModule(Database::class);
+        /** @var Database $db */
+        $dumpfilePath = Files::concatenatePaths([Configuration::dataDir(), 'localDatabase.sql']);
+
+        $this->debug('restoring development database');
+
+        $restoreCommand = sprintf(
+            'mysql --host=%s --user=%s --password=%s %s < %s',
+            $db->_getConfig('user'),
+            $db->_getConfig('user'),
+            $db->_getConfig('user'),
+            $db->_getConfig('user'),
+            $dumpfilePath
+        );
+
+        $cli->runShellCommand($restoreCommand);
+        $filesystem = $this->getModule('Filesystem');
+        /** @var Filesystem $filesystem */
+        $filesystem->deleteFile($dumpfilePath);
+    }
+}

Tests/acceptance.suite.yml

+# Codeception Test Suite Configuration
+#
+# Suite for acceptance tests.
+# Perform tests in browser using the WebDriver or PhpBrowser.
+# If you need both WebDriver and PHPBrowser tests - create a separate suite.
+
+actor: AcceptanceTester
+modules:
+  enabled:
+    # order matters - \Helper\Acceptance has to be first!
+    - \Helper\Acceptance
+    - Filesystem
+    - Cli
+    - WebDriver:
+        browser: chrome
+        url: https://ter.ddev.site/
+        restart: true
+        window_size: 1920x2080
+        host: chromedriver
+        path: /
+        capabilities:
+          chromeOptions:
+            args:
+              - '--headless'
+              - '--disable-gpu'
+              - '--disable-dev-shm-usage'
+              - '--no-sandbox'
+    - PunktDe\Codeception\Database\Module\Database:
+        dsn: 'mysql:host=db;dbname=db;charset=UTF8'
+        user: 'db'
+        password: 'db'
+        host: 'db'
+        dump: 'Tests/_data/Dumps/Fixture.sql'
+        populate: true
+        cleanup: true
+        populator: 'mysql --host=$host --user=$user --password=$password $dbName < $dump'
+  step_decorators: ~

composer.json

@@ -23,37 +23,47 @@
         {
             "type": "vcs",
             "url": "https://git-t3o.typo3.org/t3o/gitlab_api.git"
+        },
+        {
+            "type": "vcs",
+            "url": "https://git-t3o.typo3.org/t3o/test-helpers.git"
         }
     ],
     "require": {
-        "php": ">=7.0",
-        "ext-pdo": "*",
+        "php": "^7.4",
         "ext-dom": "*",
+        "ext-json": "*",
+        "ext-pdo": "*",
+        "ext-zip": "*",
         "ext-zlib": "*",
-        "t3o/ter-layout": "^0.1",
-        "t3o/ter-soap": "^2.0",
-        "t3o/ter-frontend": "^0.4",
-        "helhum/typo3-console": "^5.7",
-        "apache-solr-for-typo3/solr": "^10.0",
-        "t3o/t3olayout": "^4.0",
-        "nikic/php-parser": "^4.0",
+        "apache-solr-for-typo3/solr": "^11.0",
         "gordalina/cachetool": "^4.0",
-        "helhum/typo3-secure-web": "^0.2.9",
-        "typo3/cms-adminpanel": "^9.5",
-        "typo3/cms-seo": "^9.5",
-        "ext-json": "*"
+        "nimut/testing-framework": "^5.0",
+        "t3o/t3olayout": "^5.0",
+        "t3o/ter-frontend": "^0.5.0",
+        "t3o/ter-layout": "^0.2.0",
+        "t3o/ter-soap": "^2.1",
+        "typo3/cms-adminpanel": "^10.4",
+        "typo3/cms-seo": "^10.4"
     },
     "require-dev": {
-        "nimut/testing-framework": "^4.0",
         "codeception/codeception": "^4.1",
         "codeception/module-phpbrowser": "^1.0",
-        "codeception/module-asserts": "^1.2"
+        "codeception/module-asserts": "^1.2",
+        "neos/utility-files": "^6.0",
+        "neos/utility-arrays": "^6.0",
+        "punktde/codeception-cli": "^2.0",
+        "punktde/codeception-webdriver": "^2.2",
+        "punktde/codeception-filesystem": "^2.0",
+        "punktde/codeception-database": "^2.0",
+        "t3o/test-helpers": "dev-master"
     },
     "config": {
         "store-auths": false,
         "platform": {
-            "php": "7.2.13"
-        }
+            "php": "7.4.8"
+        },
+        "sort-packages": true
     },
     "extra": {
         "typo3/cms": {
@@ -63,7 +73,7 @@
     },
     "scripts": {
         "test:unit": [
-            "./vendor/bin/phpunit -c .gitlab-ci/Tests/UnitTests.xml"
+            "./vendor/bin/phpunit -c .gitlab-ci/Tests/UnitTests.xml --log-junit build/junit-report.xml --coverage-text --colors=never"
         ],
         "test:api": [
             "./vendor/bin/codecept run api --steps"

extensions/ter/Classes/Api/ExtensionKey.php

@@ -11,6 +11,7 @@ namespace T3o\Ter\Api;
  * of the License, or any later version.
  */
 
+use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
 use T3o\Ter\Exception\ExtensionKeyAlreadyInUseException;
 use T3o\Ter\Exception\ExtensionKeyNotFoundException;
 use T3o\Ter\Exception\InternalServerErrorException;
@@ -20,6 +21,7 @@ use T3o\Ter\Exception\UserNotFoundException;
 use T3o\Ter\Exception\VersionExistsException;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Database\Query\Restriction\FrontendRestrictionContainer;
+use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
@@ -61,12 +63,12 @@ class ExtensionKey
     public function isRegistered(): bool
     {
         $cleanedExtensionKey = str_replace('_', '', $this->extensionKey);
-        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tx_ter_extensionkeys');
-        $result = $queryBuilder->select('extensionkey')
-            ->from('tx_ter_extensionkeys')
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tx_terfe2_domain_model_extension');
+        $result = $queryBuilder->select('ext_key')
+            ->from('tx_terfe2_domain_model_extension')
             ->add(
                 'where',
-                'REPLACE(extensionkey, "_", "") = ' . $queryBuilder->createNamedParameter($cleanedExtensionKey)
+                'REPLACE(ext_key, "_", "") = ' . $queryBuilder->createNamedParameter($cleanedExtensionKey)
             )
             ->execute()
             ->rowCount();
@@ -76,12 +78,12 @@ class ExtensionKey
 
     public function getOwner(): ?string
     {
-        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tx_ter_extensionkeys');
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tx_terfe2_domain_model_extension');
         $ownerUserName = $queryBuilder
-            ->select('ownerusername')
-            ->from('tx_ter_extensionkeys')
+            ->select('frontend_user')
+            ->from('tx_terfe2_domain_model_extension')
             ->where(
-                $queryBuilder->expr()->eq('extensionkey', $queryBuilder->createNamedParameter($this->extensionKey))
+                $queryBuilder->expr()->eq('ext_key', $queryBuilder->createNamedParameter($this->extensionKey))
             )
             ->execute()
             ->fetchColumn();
@@ -117,8 +119,6 @@ class ExtensionKey
      * Add a new extension key.
      *
      * @param ApiUser $authenticatedUser
-     * @param string $title
-     * @param string $description
      * @param string $ownerUserName
      * @return bool
      * @throws ExtensionKeyAlreadyInUseException
@@ -127,12 +127,8 @@ class ExtensionKey
      * @throws UnauthorizedException
      * @throws UserNotFoundException
      */
-    public function registerKey(
-        ApiUser $authenticatedUser,
-        string $title,
-        string $description,
-        string $ownerUserName
-    ) {
+    public function registerKey(ApiUser $authenticatedUser, string $ownerUserName)
+    {
         if (!$authenticatedUser->isAuthenticated()) {
             throw new UnauthorizedException('Access denied.', ResultCodes::ERROR_GENERAL_USERNOTFOUND);
         }
@@ -159,13 +155,14 @@ class ExtensionKey
             'pid' => GeneralUtility::makeInstance(Configuration::class)->getStoragePid(),
             'tstamp' => $GLOBALS['EXEC_TIME'],
             'crdate' => $GLOBALS['EXEC_TIME'],
-            'extensionkey' => $this->extensionKey,
-            'title' => mb_strcut($title, 0, 50, 'utf-8'),
-            'description' => mb_strcut($description, 0, 255, 'utf-8'),
-            'ownerusername' => $ownerUserName
+            'ext_key' => $this->extensionKey,
+            'frontend_user' => $ownerUserName,
+            'last_upload' => 0,
+            'versions' => 0,
+            'last_version' => 0,
         ];
-        $conn = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_ter_extensionkeys');
-        $result = $conn->insert('tx_ter_extensionkeys', $extensionKeysRow);
+        $conn = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_terfe2_domain_model_extension');
+        $result = $conn->insert('tx_terfe2_domain_model_extension', $extensionKeysRow);
         if ($result === 0) {
             throw new InternalServerErrorException(
                 'Database error while inserting extension key.',
@@ -206,11 +203,11 @@ class ExtensionKey
             throw new UserNotFoundException('The user is not an active typo3.org user', ResultCodes::ERROR_GENERAL_USERNOTFOUND);
         }
 
-        $conn = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_ter_extensionkeys');
+        $conn = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_terfe2_domain_model_extension');
         $affectedRows = $conn->update(
-            'tx_ter_extensionkeys',
-            ['ownerusername' => $newOwnerName],
-            ['extensionkey' => $this->extensionKey]
+            'tx_terfe2_domain_model_extension',
+            ['frontend_user' => $newOwnerName],
+            ['ext_key' => $this->extensionKey]
         );
         if (!$affectedRows) {
             throw new InternalServerErrorException(
@@ -234,14 +231,63 @@ class ExtensionKey
             throw new UnauthorizedException('Access denied.', ResultCodes::ERROR_DELETEEXTENSIONKEY_ACCESSDENIED);
         }
 
-        $conn = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_ter_extensions');
-        $items = $conn->select(['*'], 'tx_ter_extensions', ['extensionkey' => $this->extensionKey])->fetchAll();
-