[FEATURE] Enable SQLite in installation process 63/55563/12
authorBenni Mack <benni@typo3.org>
Mon, 5 Feb 2018 12:26:50 +0000 (13:26 +0100)
committerSusanne Moog <susanne.moog@typo3.org>
Thu, 14 Jun 2018 08:06:15 +0000 (10:06 +0200)
The patch adds SQLite as new DBMS platform to the TYPO3
instance installer if pdo_sqlite is available.

* sqlite has no database name and user / password restriction
  but stores the database in a single file.

* the filename contains a random string so it can't be easily
  guessed if the config directory is within web document root
  and the web server is configured to deliver .sqlite files.

* the feature .rst file mentions possible security risks comes
  with having a database within document root and documents
  how to prevent those.

* similar to mysql and postgres, an acceptance test verifies
  the system can be successfully installed using a blank
  installation and using the introduction package.

* bamboo plan spec is adapted to execute the sqlite installer suite

* testing-framework is raised to 3.8.1 supporting the ac test:
  composer update typo3/testing-framework

Resolves: #85256
Releases: master
Change-Id: I91a8c98f868b5e29bee4ad7dedd3cc8c50346452
Reviewed-on: https://review.typo3.org/55563
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: Benni Mack <benni@typo3.org>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
12 files changed:
Build/bamboo/src/main/java/core/AbstractCoreSpec.java
Build/bamboo/src/main/java/core/NightlySpec.java
Build/bamboo/src/main/java/core/PreMergeSpec.java
composer.lock
typo3/sysext/core/Documentation/Changelog/master/Feature-85256-InstallTYPO3OnSQLite.rst [new file with mode: 0644]
typo3/sysext/core/Tests/AcceptanceInstallMysql/InstallWithMysqlIntroductionPackageCest.php
typo3/sysext/core/Tests/AcceptanceInstallPgsql/InstallWithPgsqlIntroductionPackageCest.php
typo3/sysext/core/Tests/AcceptanceInstallSqlite.suite.yml [new file with mode: 0644]
typo3/sysext/core/Tests/AcceptanceInstallSqlite/InstallWithSqliteBlankPageCest.php [new file with mode: 0644]
typo3/sysext/core/Tests/AcceptanceInstallSqlite/InstallWithSqliteIntroductionPackageCest.php [new file with mode: 0644]
typo3/sysext/install/Classes/Controller/InstallerController.php
typo3/sysext/install/Resources/Private/Templates/Installer/ShowDatabaseConnect.html

index 43cf2e6..442553b 100644 (file)
@@ -173,7 +173,7 @@ abstract public class AbstractCoreSpec {
      */
     protected Job getJobAcceptanceTestInstallMysql(Requirement requirement, String requirementIdentifier) {
         return new Job("Accept inst my " + requirementIdentifier, new BambooKey("ACINSTMY" + requirementIdentifier))
-            .description("Install TYPO3 on mysql and create empty frontend page " + requirementIdentifier)
+            .description("Install TYPO3 on mysql and load introduction package " + requirementIdentifier)
             .pluginConfigurations(this.getDefaultJobPluginConfiguration())
             .tasks(
                 this.getTaskGitCloneRepository(),
@@ -242,6 +242,43 @@ abstract public class AbstractCoreSpec {
     }
 
     /**
+     * Job acceptance test installs system and introduction package on sqlite
+     *
+     * @param Requirement requirement
+     * @param String requirementIdentfier
+     */
+    protected Job getJobAcceptanceTestInstallSqlite(Requirement requirement, String requirementIdentifier) {
+        return new Job("Accept inst sq " + requirementIdentifier, new BambooKey("ACINSTSQ" + requirementIdentifier))
+        .description("Install TYPO3 on sqlite and load introduction package " + requirementIdentifier)
+        .pluginConfigurations(this.getDefaultJobPluginConfiguration())
+        .tasks(
+            this.getTaskGitCloneRepository(),
+            this.getTaskGitCherryPick(),
+            this.getTaskComposerInstall(),
+            this.getTaskPrepareAcceptanceTest(),
+            new CommandTask()
+                .description("Execute codeception AcceptanceInstallSqlite suite")
+                .executable("codecept")
+                .argument("run AcceptanceInstallSqlite -d -c " + this.testingFrameworkBuildPath + "AcceptanceTestsInstallSqlite.yml --xml reports.xml --html reports.html")
+                .environmentVariables(this.credentialsSqlite)
+        )
+        .finalTasks(
+            new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
+                .resultDirectories("typo3temp/var/tests/AcceptanceReportsInstallSqlite/reports.xml"),
+            this.getTaskTearDownAcceptanceTestSetup()
+        )
+        .requirements(
+            requirement
+        )
+        .artifacts(new Artifact()
+            .name("Test Report")
+            .copyPattern("typo3temp/var/tests/AcceptanceReportsInstallSqlite/")
+            .shared(false)
+        )
+        .cleanWorkingDirectory(true);
+    }
+
+    /**
      * Jobs for mysql based acceptance tests
      *
      * @param int numberOfChunks
index 2f90f98..9279792 100644 (file)
@@ -79,8 +79,8 @@ public class NightlySpec extends AbstractCoreSpec {
         ArrayList<Job> jobsMainStage = new ArrayList<Job>();
 
         jobsMainStage.add(this.getJobAcceptanceTestInstallMysql(this.getRequirementPhpVersion72(), "PHP72"));
-
         jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql(this.getRequirementPhpVersion72(), "PHP72"));
+        jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite(this.getRequirementPhpVersion72(), "PHP72"));
 
         jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, this.getRequirementPhpVersion72(), "PHP72"));
 
index 8d14cd3..962c8de 100644 (file)
@@ -91,6 +91,7 @@ public class PreMergeSpec extends AbstractCoreSpec {
 
         jobsMainStage.add(this.getJobAcceptanceTestInstallMysql(this.getRequirementPhpVersion72(), "PHP72"));
         jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql(this.getRequirementPhpVersion72(), "PHP72"));
+        jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite(this.getRequirementPhpVersion72(), "PHP72"));
 
         jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, this.getRequirementPhpVersion72(), "PHP72"));
 
index 55cd3c9..8a73e57 100644 (file)
@@ -1,7 +1,7 @@
 {
     "_readme": [
         "This file locks the dependencies of your project to a known state",
-        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
+        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
     "content-hash": "1f43a0302080cbd28f381130f2df42bb",
         },
         {
             "name": "typo3/testing-framework",
-            "version": "3.8.0",
+            "version": "3.8.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/TYPO3/testing-framework.git",
-                "reference": "987bba68fdd93a9c1fcb397c7f0711e4f2cd9d1a"
+                "reference": "024448439a037445f712ed9b60a3da293d594aaf"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/TYPO3/testing-framework/zipball/987bba68fdd93a9c1fcb397c7f0711e4f2cd9d1a",
-                "reference": "987bba68fdd93a9c1fcb397c7f0711e4f2cd9d1a",
+                "url": "https://api.github.com/repos/TYPO3/testing-framework/zipball/024448439a037445f712ed9b60a3da293d594aaf",
+                "reference": "024448439a037445f712ed9b60a3da293d594aaf",
                 "shasum": ""
             },
             "require": {
                 "tests",
                 "typo3"
             ],
-            "time": "2018-06-10T17:26:48+00:00"
+            "time": "2018-06-13T17:03:55+00:00"
         },
         {
             "name": "webmozart/assert",
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Feature-85256-InstallTYPO3OnSQLite.rst b/typo3/sysext/core/Documentation/Changelog/master/Feature-85256-InstallTYPO3OnSQLite.rst
new file mode 100644 (file)
index 0000000..6271b27
--- /dev/null
@@ -0,0 +1,45 @@
+.. include:: ../../Includes.txt
+
+=========================================
+Feature: #85256 - Install TYPO3 on SQLite
+=========================================
+
+See :issue:`85256`
+
+Description
+===========
+
+The TYPO3 web installer allows to install the system on `SQLite` DBMS.
+
+This platform can be selected if :php:`pdo_sqlite` is available in PHP, which is
+often the case. SQLite can be a nice DBMS for relatively small instances and has
+the advantage that no further server side daemon is needed.
+
+Administrators must keep an eye on security if using this platform:
+
+In SQLite, a database is stored in a single file. In TYPO3, its default location
+is the var/sqlite path of the instance which is derived from environment variable
+:php:`TYPO3_PATH_APP`. If that variable is **not** set which is
+often the case in non-composer instances, **the database file will end up in the
+web server accessible document root directory :file:`typo3conf/`**!
+
+To prevent guessing the database name and simply downloading it, the installer appends
+a random string to the database filename during installation. Additionally, the demo
+Apache :file:`_.htaccess` file prevents downloading :file:`.sqlite` files. The demo
+MicroSoft IIS web server configuration in file :file:`_web.config` comes with the same
+restriction.
+
+Administrators installing TYPO3 using the SQLite platform should thus test if the
+database is downloadable from the web and take measures to prevent this by either
+configuring the web server to deny this file, or - better - by moving the config folder
+out of the document root, which is good practice anyway.
+
+
+Impact
+======
+
+TYPO3 can be installed to run on SQLite. If choosing this option, administrators
+must check the file is never delivered by the web server.
+
+
+.. index:: Database
index b28a3ff..062659b 100644 (file)
@@ -70,7 +70,7 @@ class InstallWithMysqlIntroductionPackageCest
         $I->switchToIFrame('list_frame');
         $I->waitForText('Get preconfigured distribution', 30);
         $I->click('.t3-button-action-installdistribution');
-        $I->waitForText('You successfully installed the distribution \'introduction\'', 120);
+        $I->waitForText('You successfully installed the distribution \'introduction\'', 240);
 
         // Verify default frontend is rendered
         $I->amOnPage('/');
index 074b3e7..4024c1f 100644 (file)
@@ -66,7 +66,7 @@ class InstallWithPgsqlIntroductionPackageCest
         $I->switchToIFrame('list_frame');
         $I->waitForText('Get preconfigured distribution', 30);
         $I->click('.t3-button-action-installdistribution');
-        $I->waitForText('You successfully installed the distribution \'introduction\'', 120);
+        $I->waitForText('You successfully installed the distribution \'introduction\'', 240);
 
         // Verify default frontend is rendered
         $I->amOnPage('/');
diff --git a/typo3/sysext/core/Tests/AcceptanceInstallSqlite.suite.yml b/typo3/sysext/core/Tests/AcceptanceInstallSqlite.suite.yml
new file mode 100644 (file)
index 0000000..14b7e09
--- /dev/null
@@ -0,0 +1,21 @@
+class_name: AcceptanceTester
+modules:
+  enabled:
+    - WebDriver
+    - \Helper\Acceptance
+    - Asserts
+  config:
+    WebDriver:
+      url: http://localhost:8000/typo3temp/var/tests/acceptanceinstallsqlite
+      browser: chrome
+      port: 9515
+      capabilities:
+        # Disable the "scroll to element before clicking" behavior as this breaks tests
+        # where for example a fixed docbar is used. Selenium scrolls to the element before
+        # clicking it and then complains that it can't click the element because another elemnt
+        # is overlaying it.
+        # You have to ensure that the element is in the viewport by your own before clicking it!
+        # You can simply do that by scrolling to it.
+        elementScrollBehavior: 1
+        chromeOptions:
+          args: ["--headless", "--no-sandbox", "window-size=1280x1024", "--proxy-server='direct://'", "--proxy-bypass-list=*", "--disable-gpu"]
diff --git a/typo3/sysext/core/Tests/AcceptanceInstallSqlite/InstallWithSqliteBlankPageCest.php b/typo3/sysext/core/Tests/AcceptanceInstallSqlite/InstallWithSqliteBlankPageCest.php
new file mode 100644 (file)
index 0000000..523e381
--- /dev/null
@@ -0,0 +1,65 @@
+<?php
+namespace TYPO3\CMS\Core\Tests\AcceptanceInstallSqlite;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * Click through installer, go to backend, check blank site in FE works
+ */
+class InstallWithSqliteBlankPageCest
+{
+    /**
+     * @param \AcceptanceTester $I
+     */
+    public function installTypo3OnSqlite(\AcceptanceTester $I)
+    {
+        // Calling frontend redirects to installer
+        $I->amOnPage('/');
+
+        // EnvironmentAndFolders step
+        $I->waitForText('Installing TYPO3');
+        $I->waitForText('No problems detected, continue with installation');
+        $I->click('No problems detected, continue with installation');
+
+        // DatabaseConnection step
+        $I->waitForText('Select database');
+        $I->selectOption('#t3js-connect-database-driver', 'Manually configured SQLite connection');
+        $I->click('Continue');
+
+        // DatabaseData step
+        $I->waitForText('Create Administrative User / Specify Site Name');
+        $I->fillField('#username', 'admin');
+        $I->fillField('#password', 'password');
+        $I->click('Continue');
+
+        // DefaultConfiguration step - load distributions
+        $I->waitForText('Installation Complete');
+        $I->click('#create-site');
+        $I->click('Open the TYPO3 Backend');
+
+        // Verify backend login successful
+        $I->waitForElement('#t3-username');
+        $I->fillField('#t3-username', 'admin');
+        $I->fillField('#t3-password', 'password');
+        $I->click('#t3-login-submit-section > button');
+        $I->waitForElement('.nav', 30);
+        $I->waitForElement('.scaffold-content iframe', 30);
+        $I->seeCookie('be_lastLoginProvider');
+        $I->seeCookie('be_typo_user');
+
+        // Verify default frontend is rendered
+        $I->amOnPage('/');
+        $I->waitForText('Welcome to a default website made with TYPO3');
+    }
+}
diff --git a/typo3/sysext/core/Tests/AcceptanceInstallSqlite/InstallWithSqliteIntroductionPackageCest.php b/typo3/sysext/core/Tests/AcceptanceInstallSqlite/InstallWithSqliteIntroductionPackageCest.php
new file mode 100644 (file)
index 0000000..bbaff51
--- /dev/null
@@ -0,0 +1,77 @@
+<?php
+namespace TYPO3\CMS\Core\Tests\AcceptanceInstallSqlite;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * Click through installer, go to backend, install introduction package
+ */
+class InstallWithSqliteIntroductionPackageCest
+{
+    /**
+     * @param \AcceptanceTester $I
+     */
+    public function installTypo3OnSqlite(\AcceptanceTester $I)
+    {
+        // Calling frontend redirects to installer
+        $I->amOnPage('/');
+
+        // EnvironmentAndFolders step
+        $I->waitForText('Installing TYPO3');
+        $I->waitForText('No problems detected, continue with installation');
+        $I->click('No problems detected, continue with installation');
+
+        // DatabaseConnection step
+        $I->waitForText('Select database');
+        $I->selectOption('#t3js-connect-database-driver', 'Manually configured SQLite connection');
+        $I->click('Continue');
+
+        // DatabaseData step
+        $I->waitForText('Create Administrative User / Specify Site Name');
+        $I->fillField('#username', 'admin');
+        $I->fillField('#password', 'password');
+        $I->click('Continue');
+
+        // DefaultConfiguration step - load distributions
+        $I->waitForText('Installation Complete');
+        $I->click('#load-distributions');
+        $I->click('Open the TYPO3 Backend');
+
+        // Verify backend login successful
+        $I->waitForElement('#t3-username');
+        $I->fillField('#t3-username', 'admin');
+        $I->fillField('#t3-password', 'password');
+        $I->click('#t3-login-submit-section > button');
+        $I->waitForElement('.nav', 30);
+        $I->waitForElement('.scaffold-content iframe', 30);
+        $I->seeCookie('be_lastLoginProvider');
+        $I->seeCookie('be_typo_user');
+
+        // Loading might take some time
+        $I->wait(10);
+        $I->switchToIFrame('list_frame');
+        $I->waitForText('Get preconfigured distribution', 30);
+        $I->click('.t3-button-action-installdistribution');
+        $I->waitForText('You successfully installed the distribution \'introduction\'', 240);
+
+        // Verify default frontend is rendered
+        $I->amOnPage('/');
+        $I->waitForText('Let us introduce you to TYPO3', 30);
+        $I->waitForText('Make it your own');
+
+        // Verify link
+        $I->click('[title="Features"]');
+        $I->waitForText('Feature Complete Out-of-the-box', 30);
+    }
+}
index 07b2d6d..7e9ab09 100644 (file)
@@ -23,6 +23,7 @@ use TYPO3\CMS\Core\Configuration\ConfigurationManager;
 use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
 use TYPO3\CMS\Core\Core\Bootstrap;
 use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\Crypto\Random;
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Database\Schema\Exception\StatementException;
@@ -299,6 +300,17 @@ class InstallerController
                 $activeAvailableOption = 'postgresManualConfiguration';
             }
         }
+        if (extension_loaded('pdo_sqlite')) {
+            $hasAtLeastOneOption = true;
+            $view->assign('hasSqliteManualConfiguration', true);
+            $view->assign(
+                'sqliteManualConfigurationOptions',
+                []
+            );
+            if ($GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['driver'] === 'pdo_sqlite') {
+                $activeAvailableOption = 'sqliteManualConfiguration';
+            }
+        }
 
         if (!empty($this->getDatabaseConfigurationFromEnvironment())) {
             $hasAtLeastOneOption = true;
@@ -339,6 +351,7 @@ class InstallerController
                     'pdo_mysql',
                     'pdo_pgsql',
                     'mssql',
+                    'pdo_sqlite',
                 ];
                 if (in_array($postValues['driver'], $validDrivers, true)) {
                     $defaultConnectionSettings['driver'] = $postValues['driver'];
@@ -412,6 +425,18 @@ class InstallerController
                     );
                 }
             }
+            // For sqlite a db path is automatically calculated
+            if (isset($postValues['driver']) && $postValues['driver'] === 'pdo_sqlite') {
+                $dbFilename = '/cms-' . (new Random())->generateRandomHexString(8) . '.sqlite';
+                // If the var/ folder exists outside of document root, put it into var/sqlite/
+                // Otherwise simply into typo3conf/
+                if (Environment::getProjectPath() !== Environment::getPublicPath()) {
+                    GeneralUtility::mkdir_deep(Environment::getVarPath() . '/sqlite');
+                    $defaultConnectionSettings['path'] = Environment::getVarPath() . '/sqlite' . $dbFilename;
+                } else {
+                    $defaultConnectionSettings['path'] = Environment::getConfigPath() . $dbFilename;
+                }
+            }
         }
 
         $success = false;
@@ -455,7 +480,9 @@ class InstallerController
     public function checkDatabaseSelectAction(): ResponseInterface
     {
         $success = false;
-        if ((string)$GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['dbname'] !== '') {
+        if ((string)$GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['dbname'] !== ''
+            || (string)$GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['path'] !== ''
+        ) {
             try {
                 $success = GeneralUtility::makeInstance(ConnectionPool::class)
                     ->getConnectionByName(ConnectionPool::DEFAULT_CONNECTION_NAME)
@@ -815,6 +842,7 @@ For each website you need a TypoScript template on the main page of your website
     /**
      * Check LocalConfiguration.php for required database settings:
      * - 'username' and 'password' are mandatory, but may be empty
+     * - if 'driver' is pdo_sqlite and 'path' is set, its ok, too
      *
      * @return bool TRUE if required settings are present
      */
@@ -827,6 +855,12 @@ For each website you need a TypoScript template on the main page of your website
         if (!isset($GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['password'])) {
             $configurationComplete = false;
         }
+        if (isset($GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['driver'])
+            && $GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['driver'] === 'pdo_sqlite'
+            && !empty($GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default']['path'])
+        ) {
+            $configurationComplete = true;
+        }
         return $configurationComplete;
     }
 
index c9fa376..b54849c 100644 (file)
                                                                        Manually configured PostgreSQL connection
                                                                        </option>
                                                                </f:if>
+                                                               <f:if condition="{hasSqliteManualConfiguration}">
+                                                                       <option
+                                                                               value="sqliteManualConfiguration"
+                                                                               {f:if(condition:'{activeAvailableOption} == sqliteManualConfiguration', then: 'selected="selected"')}
+                                                                       >
+                                                                       Manually configured SQLite connection
+                                                                       </option>
+                                                               </f:if>
                                                                <f:if condition="{hasConfigurationFromEnvironment}">
                                                                        <option
                                                                                value="configurationFromEnvironment"
                                                </div>
                                        </f:if>
 
+                                       <f:if condition="{hasSqliteManualConfiguration}">
+                                               <div id="sqliteManualConfiguration" class="t3-install-driver-data">
+                                                       <input type="hidden" value="pdo_sqlite" name="install[values][driver]"/>
+                                               </div>
+                                       </f:if>
+
                                        <f:if condition="{hasConfigurationFromEnvironment}">
                                                <div id="configurationFromEnvironment" class="t3-install-driver-data">
                                                </div>