[TASK] Only import extensions from 2015+ into EM
[Packages/TYPO3.CMS.git] / typo3 / sysext / extensionmanager / Classes / Utility / Importer / ExtensionListUtility.php
1 <?php
2 namespace TYPO3\CMS\Extensionmanager\Utility\Importer;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16 use TYPO3\CMS\Core\Database\ConnectionPool;
17 use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
18 use TYPO3\CMS\Core\Utility\GeneralUtility;
19 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
20 use TYPO3\CMS\Extensionmanager\Utility\Parser\AbstractExtensionXmlParser;
21
22 /**
23 * Importer object for extension list
24 */
25 class ExtensionListUtility implements \SplObserver
26 {
27 /**
28 * Keeps instance of a XML parser.
29 *
30 * @var AbstractExtensionXmlParser
31 */
32 protected $parser;
33
34 /**
35 * Keeps number of processed version records.
36 *
37 * @var int
38 */
39 protected $sumRecords = 0;
40
41 /**
42 * Keeps record values to be inserted into database.
43 *
44 * @var array
45 */
46 protected $arrRows = [];
47
48 /**
49 * Keeps fieldnames of tx_extensionmanager_domain_model_extension table.
50 *
51 * @var array
52 */
53 protected static $fieldNames = [
54 'extension_key',
55 'version',
56 'integer_version',
57 'current_version',
58 'alldownloadcounter',
59 'downloadcounter',
60 'title',
61 'ownerusername',
62 'author_name',
63 'author_email',
64 'authorcompany',
65 'last_updated',
66 'md5hash',
67 'repository',
68 'state',
69 'review_state',
70 'category',
71 'description',
72 'serialized_dependencies',
73 'update_comment'
74 ];
75
76 /**
77 * Table name to be used to store extension models.
78 *
79 * @var string
80 */
81 protected static $tableName = 'tx_extensionmanager_domain_model_extension';
82
83 /**
84 * Maximum of rows that can be used in a bulk insert for the current
85 * database platform.
86 *
87 * @var int
88 */
89 protected $maxRowsPerChunk = 50;
90
91 /**
92 * Keeps indexes of fields that should not be quoted.
93 *
94 * @var array
95 */
96 protected static $fieldIndicesNoQuote = [2, 3, 5, 11, 13, 14, 15, 16];
97
98 /**
99 * Keeps repository UID.
100 *
101 * The UID is necessary for inserting records.
102 *
103 * @var int
104 */
105 protected $repositoryUid = 1;
106
107 /**
108 * @var \TYPO3\CMS\Extensionmanager\Domain\Repository\RepositoryRepository
109 */
110 protected $repositoryRepository;
111
112 /**
113 * @var \TYPO3\CMS\Extensionmanager\Domain\Repository\ExtensionRepository
114 */
115 protected $extensionRepository;
116
117 /**
118 * @var \TYPO3\CMS\Extensionmanager\Domain\Model\Extension
119 */
120 protected $extensionModel;
121
122 /**
123 * @var \TYPO3\CMS\Extbase\Object\ObjectManager
124 */
125 protected $objectManager;
126
127 /**
128 * Only import extensions newer than this date (timestamp),
129 * see constructor
130 *
131 * @var int
132 */
133 protected $minimumDateToImport;
134
135 /**
136 * Class constructor.
137 *
138 * Method retrieves and initializes extension XML parser instance.
139 *
140 * @throws \TYPO3\CMS\Extensionmanager\Exception\ExtensionManagerException
141 */
142 public function __construct()
143 {
144 /** @var $objectManager \TYPO3\CMS\Extbase\Object\ObjectManager */
145 $this->objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\ObjectManager::class);
146 $this->repositoryRepository = $this->objectManager->get(\TYPO3\CMS\Extensionmanager\Domain\Repository\RepositoryRepository::class);
147 $this->extensionRepository = $this->objectManager->get(\TYPO3\CMS\Extensionmanager\Domain\Repository\ExtensionRepository::class);
148 $this->extensionModel = $this->objectManager->get(\TYPO3\CMS\Extensionmanager\Domain\Model\Extension::class);
149 // @todo catch parser exception
150 $this->parser = \TYPO3\CMS\Extensionmanager\Utility\Parser\XmlParserFactory::getParserInstance('extension');
151 if (is_object($this->parser)) {
152 $this->parser->attach($this);
153 } else {
154 throw new \TYPO3\CMS\Extensionmanager\Exception\ExtensionManagerException(
155 static::class . ': No XML parser available.',
156 1476108717
157 );
158 }
159
160 $connection = GeneralUtility::makeInstance(ConnectionPool::class)
161 ->getConnectionForTable(self::$tableName);
162 $maxBindParameters = PlatformInformation::getMaxBindParameters(
163 $connection->getDatabasePlatform()
164 );
165 $countOfBindParamsPerRow = count(self::$fieldNames);
166 // flush at least chunks of 50 elements - in case the currently used
167 // database platform does not support that, the threshold is lowered
168 $this->maxRowsPerChunk = min(
169 $this->maxRowsPerChunk,
170 floor($maxBindParameters / $countOfBindParamsPerRow)
171 );
172 // Only import extensions that are compatible with 7.6 or higher.
173 // TER only allows to publish extensions with compatibility if the TYPO3 version has been released
174 // And 7.6 was released on 10th of November 2015.
175 // This effectively reduces the number of extensions imported into this TYPO3 installation
176 // by more than 70%. As long as the extensions.xml from TER includes these files, we need to "hack" this
177 // within TYPO3 Core.
178 // For TYPO3 v10.0, this date could be set to 2017-04-04 (8 LTS release).
179 // Also see https://decisions.typo3.org/t/reduce-size-of-extension-manager-db-table/329/
180 $this->minimumDateToImport = strtotime('2015-11-10T00:00:00+00:00');
181 }
182
183 /**
184 * Method initializes parsing of extension.xml.gz file.
185 *
186 * @param string $localExtensionListFile absolute path to extension list xml.gz
187 * @param int $repositoryUid UID of repository when inserting records into DB
188 * @return int total number of imported extension versions
189 */
190 public function import($localExtensionListFile, $repositoryUid = null)
191 {
192 if ($repositoryUid !== null && is_int($repositoryUid)) {
193 $this->repositoryUid = $repositoryUid;
194 }
195 $zlibStream = 'compress.zlib://';
196 $this->sumRecords = 0;
197 $this->parser->parseXml($zlibStream . $localExtensionListFile);
198 // flush last rows to database if existing
199 if (!empty($this->arrRows)) {
200 GeneralUtility::makeInstance(ConnectionPool::class)
201 ->getConnectionForTable('tx_extensionmanager_domain_model_extension')
202 ->bulkInsert(
203 'tx_extensionmanager_domain_model_extension',
204 $this->arrRows,
205 self::$fieldNames
206 );
207 }
208 $extensions = $this->extensionRepository->insertLastVersion($this->repositoryUid);
209 $this->repositoryRepository->updateRepositoryCount($extensions, $this->repositoryUid);
210 return $this->sumRecords;
211 }
212
213 /**
214 * Method collects and stores extension version details into the database.
215 *
216 * @param AbstractExtensionXmlParser $subject a subject notifying this observer
217 */
218 protected function loadIntoDatabase(AbstractExtensionXmlParser &$subject)
219 {
220 if ($this->sumRecords !== 0 && $this->sumRecords % $this->maxRowsPerChunk === 0) {
221 GeneralUtility::makeInstance(ConnectionPool::class)
222 ->getConnectionForTable(self::$tableName)
223 ->bulkInsert(
224 self::$tableName,
225 $this->arrRows,
226 self::$fieldNames
227 );
228 $this->arrRows = [];
229 }
230 $versionRepresentations = VersionNumberUtility::convertVersionStringToArray($subject->getVersion());
231 // order must match that of self::$fieldNames!
232 $this->arrRows[] = [
233 $subject->getExtkey(),
234 $subject->getVersion(),
235 $versionRepresentations['version_int'],
236 // initialize current_version, correct value computed later:
237 0,
238 (int)$subject->getAlldownloadcounter(),
239 (int)$subject->getDownloadcounter(),
240 $subject->getTitle() !== null ? $subject->getTitle() : '',
241 $subject->getOwnerusername(),
242 $subject->getAuthorname() !== null ? $subject->getAuthorname() : '',
243 $subject->getAuthoremail() !== null ? $subject->getAuthoremail() : '',
244 $subject->getAuthorcompany() !== null ? $subject->getAuthorcompany() : '',
245 (int)$subject->getLastuploaddate(),
246 $subject->getT3xfilemd5(),
247 $this->repositoryUid,
248 $this->extensionModel->getDefaultState($subject->getState() ?: ''),
249 (int)$subject->getReviewstate(),
250 $this->extensionModel->getCategoryIndexFromStringOrNumber($subject->getCategory() ?: ''),
251 $subject->getDescription() ?: '',
252 $subject->getDependencies() ?: '',
253 $subject->getUploadcomment() ?: ''
254 ];
255 ++$this->sumRecords;
256 }
257
258 /**
259 * Method receives an update from a subject.
260 *
261 * @param \SplSubject $subject a subject notifying this observer
262 */
263 public function update(\SplSubject $subject)
264 {
265 if (is_subclass_of($subject, AbstractExtensionXmlParser::class)) {
266 if ((int)$subject->getLastuploaddate() > $this->minimumDateToImport) {
267 $this->loadIntoDatabase($subject);
268 }
269 }
270 }
271 }