Commit 9f84c03b authored by Tomas Norre Mikkelsen's avatar Tomas Norre Mikkelsen
Browse files

Switch model for downloads and fetch data from packagist

parent 9269545f
<?php
declare(strict_types=1);
namespace T3o\TerFe2\Command;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use T3o\TerFe2\Domain\Repository\VersionRepository;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class PackagistCommandController extends CommandController
{
/**
* @var ExtensionRepository
*/
protected $extensionRepository;
/**
* @var VersionRepository
*/
protected $versionRepository;
/**
* @param ExtensionRepository $extensionRepository
*/
public function injectExtensionRepository(ExtensionRepository $extensionRepository)
{
$this->extensionRepository = $extensionRepository;
}
public function injectVersionRepository(VersionRepository $versionRepository)
{
$this->versionRepository = $versionRepository;
}
/**
* Fetch download data from Packagist
*
* Examples:
*
* --- Fetch download data from packagist
* $ typo3cms packagist:fetchdownloaddata 10
*
* @param int $limit You can set a limit, default is 10
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public function fetchDownloadDataCommand(int $limit = 10)
{
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$this->extensionRepository->setDefaultQuerySettings($querySettings);
$this->versionRepository->setDefaultQuerySettings($querySettings);
$extensions = $this->extensionRepository->findWithComposerName($limit);
// User-Agent https://twitter.com/seldaek/status/1095420243315511297
$options = array('http' => array('user_agent' => 'Thanks a bunch! Fetching packagist Download Data for https://extensions.typo3.org - https://typo3.org/community/teams/typo3org/'));
$context = stream_context_create($options);
/** @var Extension $extension */
foreach ($extensions as $extension) {
$this->outputLine('<info> Processing: ' . $extension->getExtKey() . '</info>');
$packagistUrl = 'https://packagist.org/packages/' . $extension->getComposerName() . '/downloads.json';
/** @var Extension $extension */
$extension = $this->extensionRepository->findOneByExtKey($extension->getExtKey());
$downloadData = json_decode(file_get_contents($packagistUrl, false, $context), true);
foreach ($downloadData['package']['downloads']['versions'] as $version => $downloads) {
$versionFound = $this->versionRepository->findOneByExtensionAndVersionString($extension, $version);
if (null === $versionFound) {
continue;
}
if (!$version) {
continue;
}
VersionUtility::updateVersionCounter(
$extension->getExtKey(),
$versionFound->getUid(),
VersionUtility::DOWNLOAD_SOURCE_PACKAGIST,
$downloads['monthly']
);
}
}
}
}
\ No newline at end of file
......@@ -20,6 +20,7 @@ use T3o\Ter\Api\ExtensionVersion;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Provider\FileProvider;
use T3o\TerFe2\Utility\ExtensionUtility;
use T3o\TerFe2\Utility\VersionUtility;
use T3o\TerFe2\Validation\Validator\ComposerNameValidator;
use TYPO3\CMS\Core\Page\PageRenderer;
use TYPO3\CMS\Core\Utility\GeneralUtility;
......@@ -417,12 +418,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
if (!empty($this->settings['countDownloads'])) {
$extensionKey = $extension->getExtKey();
$downloads = $this->session->get('downloads');
$versionRepository = $this->objectManager->get(\T3o\TerFe2\Domain\Repository\VersionRepository::class);
if (empty($downloads) || !in_array($extensionKey, $downloads)) {
// Add +1 to download counter and save immediately
$version->incrementDownloadCounter();
$versionRepository->update($version);
$this->persistenceManager->persistAll();
VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), VersionUtility::DOWNLOAD_SOURCE_TER);
// Add extension key to session
$downloads[] = $extensionKey;
......
<?php
declare(strict_types=1);
namespace T3o\TerFe2\Domain\Model;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Extbase\DomainObject\AbstractEntity;
class Download extends AbstractEntity
{
/**
* @var string
*/
protected $extensionKey = '';
/**
* @var int
*/
protected $versionId = 0;
/**
* @var int
*/
protected $month = 0;
/**
* @var int
*/
protected $counter = 0;
/**
* @var int
*/
protected $source = 0;
/**
* @return string
*/
public function getExtensionKey(): string
{
return $this->extensionKey;
}
/**
* @param string $extensionKey
*/
public function setExtensionKey(string $extensionKey)
{
$this->extensionKey = $extensionKey;
}
/**
* @return int
*/
public function getVersionId(): int
{
return $this->versionId;
}
/**
* @param int $versionId
*/
public function setVersionId(int $versionId)
{
$this->versionId = $versionId;
}
/**
* @return int
*/
public function getMonth(): int
{
return $this->month;
}
/**
* @param int $month
*/
public function setMonth(int $month)
{
$this->month = $month;
}
/**
* @return int
*/
public function getCounter(): int
{
return $this->counter;
}
/**
* @param int $counter
*/
public function setCounter(int $counter)
{
$this->counter = $counter;
}
/**
* @return int
*/
public function getSource(): int
{
return $this->source;
}
/**
* @param int $source
*/
public function setSource(int $source)
{
$this->source = $source;
}
}
\ No newline at end of file
......@@ -17,6 +17,7 @@ namespace T3o\TerFe2\Domain\Model;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\VersionNumberUtility;
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use TYPO3\CMS\Extbase\Persistence\ObjectStorage;
/**
......@@ -144,6 +145,21 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
*/
protected $crowdinKey = '';
/**
* @var int
*/
protected $lastDownloadSync;
/**
* @var DownloadRepository
*/
protected $downloadRepository;
public function injectDownloadRepository(DownloadRepository $downloadRepository)
{
$this->downloadRepository = $downloadRepository;
}
/**
* Initialize all ObjectStorage instances.
*/
......@@ -567,6 +583,30 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
return json_encode($versions);
}
/**
* @return string
*/
public function getDownloadsByTimeIntervalAsJson(): string
{
if(empty($this->versions)) {
return '';
}
$intervals = [];
// The for loop is turned around to ensure the sorting order in the multi dimensional array.
for ($i = 5; $i >= 0 ; $i--) {
$time = strtotime("-" . $i . "month");
$month = date('Ym', $time);
$downloads = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyTotal($month, $this->getExtKey());
$intervals['release'][] = $month;
$intervals['downloads'][] = $downloads;
$intervals['versions'][] = '';
}
return json_encode($intervals);
}
/**
* @return \DateTime|null
*/
......@@ -735,4 +775,20 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
return array_merge($supportOlderVersions, $supportELTSVersions, $supportDevVersion);
}
/**
* @return int
*/
public function getLastDownloadSync(): int
{
return $this->lastDownloadSync;
}
/**
* @param int $lastDownloadSync
*/
public function setLastDownloadSync(int $lastDownloadSync)
{
$this->lastDownloadSync = $lastDownloadSync;
}
}
<?php
declare(strict_types=1);
namespace T3o\TerFe2\Domain\Repository;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Model\Download;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\ClassNamingUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class DownloadRepository extends AbstractRepository
{
/**
* @var string
*/
protected $tableName = 'tx_terfe2_domain_model_download';
/**
* @var QueryBuilder
*/
protected $queryBuilder = QueryBuilder::class;
/**
* QueueRepository constructor.
*/
public function __construct()
{
$this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
$this->objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$this->objectType = ClassNamingUtility::translateRepositoryNameToModelName($this->getRepositoryClassName());
}
/**
* @param $month
* @param $extensionKey
* @param $versionId
* @param $source
*
* @return object|null
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
*/
public function findDownloadCounterByMonthAndExtensionKey($month, $extensionKey, $versionId, $source)
{
$query = $this->createQuery();
$query->matching(
$query->logicalAnd(
[
$query->equals('month', $month),
$query->equals('extensionKey', $extensionKey),
$query->equals('versionId', $versionId),
$query->equals('source', $source),
]
)
);
return $query->execute()->getFirst();
}
/**
* @param $month
* @param $extensionKey
*
* @return int
*/
public function findDownloadCounterByMonthAndExtensionKeyTotal($month, $extensionKey)
{
$total = 0;
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$query = $this->createQuery();
$query->setQuerySettings($querySettings);
$query->matching(
$query->logicalAnd(
[
$query->equals('month', $month),
$query->equals('extensionKey', $extensionKey)
]
)
);
$downloads = $query->execute()->toArray();
/** @var Download $download */
foreach ($downloads as $download) {
$total = $total + $download->getCounter();
}
return $total;
}
}
\ No newline at end of file
......@@ -16,6 +16,8 @@ namespace T3o\TerFe2\Domain\Repository;
use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
use TYPO3\CMS\Extbase\Persistence\QueryInterface;
/**
* Repository for \T3o\TerFe2\Domain\Model\Extension
......@@ -218,4 +220,26 @@ class ExtensionRepository extends \T3o\TerFe2\Domain\Repository\AbstractReposito
time()
);
}
/*
* @param int $limit
* @return array|\TYPO3\CMS\Extbase\Persistence\QueryResultInterface
*/
public function findWithComposerName(int $limit = 10)
{
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$query = $this->createQuery();
$query->setQuerySettings($querySettings);
$query->matching(
$query->logicalNot(
$query->equals('composer_name', '')
)
);
$query->setOrderings(['lastDownloadSync' => QueryInterface::ORDER_ASCENDING]);
$query->setLimit($limit);
return $query->execute();
}
}
......@@ -15,13 +15,23 @@ namespace T3o\TerFe2\Utility;
*/
use T3o\TerFe2\Service\LTSVersionService;
use T3o\TerFe2\Domain\Model\Download;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
/**
* Utilities to manage versions
*/
class VersionUtility
{
const DOWNLOAD_SOURCE_TER = 1;
const DOWNLOAD_SOURCE_PACKAGIST = 2;
/**
* Returns the three part version number (string) from an integer, eg 4012003 -> '4.12.3'
*
......@@ -180,4 +190,49 @@ class VersionUtility
$parts = explode('.', $versionString) ?: [];
return (int)array_shift($parts);
}
/**
* @param String $extensionKey
* @param int $versionId
* @param int $source
* @param int $counter
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public static function updateVersionCounter(String $extensionKey, int $versionId, int $source = self::DOWNLOAD_SOURCE_TER, int $counter = 1)
{
$objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$persistenceManager = $objectManager->get(PersistenceManager::class);
$extensionRepository = $objectManager->get(ExtensionRepository::class);
$month = date('Ym');
$downloadRepository = $objectManager->get(DownloadRepository::class);
/** @var Download $downloadInfo */
$downloadInfo = $downloadRepository->findDownloadCounterByMonthAndExtensionKey($month,$extensionKey,$versionId, $source);
if (null === $downloadInfo) {
$downloadInfo = new Download();
$downloadInfo->setExtensionKey($extensionKey);
$downloadInfo->setSource($source);
$downloadInfo->setMonth($month);
$downloadInfo->setVersionId($versionId);
$downloadInfo->setCounter($counter);
$downloadRepository->add($downloadInfo);
} elseif ($downloadInfo->getCounter() > 0) {
$downloadInfo->setCounter($downloadInfo->getCounter()+1);
$downloadRepository->update($downloadInfo);
}
// Sets Extension Last Download Sync to secure oldest syncs happens first.
/** @var Extension $extension */
$extension = $extensionRepository->findOneByExtKey($extensionKey);
$extension->setLastDownloadSync(time());
$extensionRepository->update($extension);
$persistenceManager->persistAll();
}
}
<?php
return [
'ctrl' => [
'title' => 'domain_model_download',
'label' => 'extension_key',
'tstamp' => 'tstamp',
'iconfile' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extRelPath('ter_fe2') . 'Resources/Public/Icons/tag.gif',
],
'interface' => [
'showRecordFieldList' => 'extension_key,version_id',
],
'types' => [
'1' => ['showitem' => 'extension_key'],
],
'palettes' => [
'1' => ['showitem' => ''],
],
'columns' => [
'extension_key' => [
'label' => 'Extension Key',
'config' => [
'type' => 'input',
],
],
'version_id' => [
'label' => 'Version ID',
'config' => [
'type' => 'input',
],
],
'source' => [
'label' => 'Download Source',