Skip to content

GitLab

Commit 4cda7598 authored by Tomas Norre Mikkelsen's avatar Tomas Norre Mikkelsen
Browse files

[FEATURE] Add Downloads from Packagist

parent fb0ab004
Hide whitespace changes
Inline Side-by-side
extensions/ter_fe2/Classes/Command/PackagistCommandController.php → extensions/ter_fe2/Classes/Command/PackagistCommand.php
View file @ 4cda7598
......@@ -16,16 +16,27 @@ namespace T3o\TerFe2\Command;
* The TYPO3 project - inspiring people to share!
*/
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use T3o\TerFe2\Domain\Repository\VersionRepository;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class PackagistCommandController extends CommandController
class PackagistCommand extends Command
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @var ExtensionRepository
*/
......@@ -36,36 +47,48 @@ class PackagistCommandController extends CommandController
*/
protected $versionRepository;
/**
* @param ExtensionRepository $extensionRepository
*/
public function injectExtensionRepository(ExtensionRepository $extensionRepository)
public function __construct(string $name = null, ObjectManager $objectManager = null, ExtensionRepository $extensionRepository = null, VersionRepository $versionRepository = null)
{
$this->extensionRepository = $extensionRepository;
$this->objectManager = $objectManager ?? GeneralUtility::makeInstance(ObjectManager::class);
//$this->extensionRepository = $extensionRepository ?? GeneralUtility::makeInstance(ExtensionRepository::class, $this->objectManager);
//$this->versionRepository = $versionRepository ?? GeneralUtility::makeInstance(VersionRepository::class, $this->objectManager);
$this->extensionRepository = $extensionRepository ?? $this->objectManager->get(ExtensionRepository::class);
$this->versionRepository = $versionRepository ?? $this->objectManager->get(VersionRepository::class);
//$this->extensionRepository = $extensionRepository ?? new ExtensionRepository();
//$this->versionRepository = $versionRepository ?? new VersionRepository();
parent::__construct($name);
}
public function injectVersionRepository(VersionRepository $versionRepository)
protected function configure()
{
$this->versionRepository = $versionRepository;
$this->setDescription('Fetches download numbers from Packagist');
$this->addOption(
'limit',
'l',
InputArgument::OPTIONAL,
'This will set the limit of how many extension to fetch download data off',
25
);
}
/**
* Fetch download data from Packagist
*
* Examples:
*
* --- Fetch download data from packagist
* $ typo3cms packagist:fetchdownloaddata 10
*
* @param int $limit You can set a limit, default is 10
* @param InputInterface $input
* @param OutputInterface $output
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public function fetchDownloadDataCommand(int $limit = 10)
public function execute(InputInterface $input, OutputInterface $output)
{
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$io = new SymfonyStyle($input, $output);
$limit = (int) $input->getOption('limit');
$querySettings = $objectManager->get(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$this->extensionRepository->setDefaultQuerySettings($querySettings);
$this->versionRepository->setDefaultQuerySettings($querySettings);
......@@ -78,10 +101,19 @@ class PackagistCommandController extends CommandController
/** @var Extension $extension */
foreach ($extensions as $extension) {
$this->outputLine('<info> Processing: ' . $extension->getExtKey() . '</info>');
$io->writeln('<info> Processing: ' . $extension->getExtKey() . '</info>');
$packagistUrl = 'https://packagist.org/packages/' . $extension->getComposerName() . '/downloads.json';
/** @var Extension $extension */
$extension = $this->extensionRepository->findOneByExtKey($extension->getExtKey());
$this->updateLastDownloadSyncForExtension($extension);
// A Packagist URL returned 404 and would proceed, so it will be skipped for this run.
// TODO: Perhaps an report should be generated if it keeps happening?
if (!$this->doesUrlReturnOKCode($packagistUrl)) {
continue;
}
$downloadData = json_decode(file_get_contents($packagistUrl, false, $context), true);
foreach ($downloadData['package']['downloads']['versions'] as $version => $downloads) {
......@@ -102,5 +134,38 @@ class PackagistCommandController extends CommandController
);
}
}
$io->writeln('Fetching download numbers from Packagist is completed');
}
/**
* Sets Extension Last Download Sync to secure oldest syncs happens first.
* @param Extension $extension
* @throws \TYPO3\CMS\Extbase\Object\Exception
*/
private function updateLastDownloadSyncForExtension(Extension $extension): void
{
$extension->setLastDownloadSync(time());
$this->extensionRepository->update($extension);
/** @var PersistenceManager $persistenceManager */
$persistenceManager = $this->objectManager->get(PersistenceManager::class);
$persistenceManager->persistAll();
}
private function doesUrlReturnOKCode(string $url): bool
{
$handle = curl_init($url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, TRUE);
/* Get the HTML or whatever is linked in $url. */
$response = curl_exec($handle);
/* Check for 404 (file not found). */
$httpCode = curl_getinfo($handle, CURLINFO_HTTP_CODE);
if($httpCode != 200) {
return false;
}
curl_close($handle);
return true;
}
}
extensions/ter_fe2/Classes/Controller/ExtensionController.php
View file @ 4cda7598
......@@ -18,6 +18,7 @@ use T3o\Ter\Api\ApiUser;
use T3o\Ter\Api\ExtensionKey;
use T3o\Ter\Api\ExtensionVersion;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use T3o\TerFe2\Provider\FileProvider;
use T3o\TerFe2\Utility\ExtensionUtility;
use T3o\TerFe2\Utility\VersionUtility;
......@@ -50,6 +51,11 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
*/
protected $ownerRepository;
/**
* @var \T3o\TerFe2\Domain\Repository\DomainRepository
*/
protected $downloadRepository;
/**
* @var \T3o\TerFe2\Persistence\Session
*/
......@@ -105,6 +111,11 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
$this->ownerRepository = $ownerRepository;
}
public function injectDownloadRepository(\T3o\TerFe2\Domain\Repository\DownloadRepository $downloadRepository)
{
$this->downloadRepository = $downloadRepository;
}
/**
* inject session
*
......@@ -179,6 +190,7 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
$this->view->assign('owner', $owner);
$this->view->assign('extension', $extension);
$this->view->assign('versionHistory', $versionHistory);
$this->view->assign('extensionDownloads', $this->downloadRepository->findDownloadsByExtensionkey($extension->getExtKey()));
/** @var \T3o\TerFe2\Service\DocumentationService $documentationService */
$documentationService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\T3o\TerFe2\Service\DocumentationService::class);
......@@ -363,6 +375,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
*/
public function downloadAction(\T3o\TerFe2\Domain\Model\Extension $extension, $versionString = '', $format = '')
{
$downloadSource = VersionUtility::DOWNLOAD_SOURCE_TER;
if ($format === '') {
$format = 'zip';
}
......@@ -408,6 +422,7 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
// Check file hash of t3x packages
if ($format === 't3x') {
$downloadSource = VersionUtility::DOWNLOAD_SOURCE_EM;
$fileHash = \T3o\TerFe2\Utility\FileUtility::getFileHash($fileUrl);
if ($fileHash != $version->getFileHash()) {
$this->redirectWithMessage($this->translate('msg.file_hash_not_equal'), 'show', '', \TYPO3\CMS\Core\Messaging\FlashMessage::ERROR, null, null, ['extension' => $extension]);
......@@ -419,7 +434,7 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
$extensionKey = $extension->getExtKey();
$downloads = $this->session->get('downloads');
if (empty($downloads) || !in_array($extensionKey, $downloads)) {
VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), VersionUtility::DOWNLOAD_SOURCE_TER);
VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), $downloadSource);
// Add extension key to session
$downloads[] = $extensionKey;
......
extensions/ter_fe2/Classes/Domain/Model/Extension.php
View file @ 4cda7598
......@@ -19,6 +19,7 @@ use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\VersionNumberUtility;
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Extbase\Persistence\ObjectStorage;
/**
......@@ -599,9 +600,19 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
for ($i = 5; $i >= 0; $i--) {
$time = strtotime('-' . $i . 'month');
$month = date('Ym', $time);
$downloads = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyTotal($month, $this->getExtKey());
$intervals['release'][] = $month;
$intervals['downloads'][] = $downloads;
$downloadsPackagist = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyAndSource($month, $this->getExtKey(), VersionUtility::DOWNLOAD_SOURCE_PACKAGIST);
$downloadsTer = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyAndSource($month, $this->getExtKey(), VersionUtility::DOWNLOAD_SOURCE_TER);
$downloadsEm = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyAndSource($month, $this->getExtKey(), VersionUtility::DOWNLOAD_SOURCE_EM);
// How to define the start date for the feature?
if ($downloadsPackagist === 0 && $downloadsTer === 0 && $downloadsEm === 0){
continue;
}
$intervals['release'][] = date('M. Y', $time);
$intervals['downloads_packagist'][] = $downloadsPackagist;
$intervals['downloads_ter'][] = $downloadsTer;
$intervals['downloads_em'][] = $downloadsEm;
$intervals['versions'][] = '';
}
......
extensions/ter_fe2/Classes/Domain/Repository/DownloadRepository.php
View file @ 4cda7598
......@@ -46,6 +46,30 @@ class DownloadRepository extends AbstractRepository
$this->objectType = ClassNamingUtility::translateRepositoryNameToModelName($this->getRepositoryClassName());
}
public function findDownloadsByExtensionkey($extensionKey)
{
$total = 0;
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$query = $this->createQuery();
$query->setQuerySettings($querySettings);
$query->matching(
$query->equals('extensionKey', $extensionKey),
);
$downloads = $query->execute()->toArray();
/** @var Download $download */
foreach ($downloads as $download) {
$total = $total + $download->getCounter();
}
return $total;
}
/**
* @param $month
* @param $extensionKey
......@@ -79,7 +103,7 @@ class DownloadRepository extends AbstractRepository
*
* @return int
*/
public function findDownloadCounterByMonthAndExtensionKeyTotal($month, $extensionKey)
public function findDownloadCounterByMonthAndExtensionKeyAndSource($month, $extensionKey, $source)
{
$total = 0;
......@@ -93,7 +117,8 @@ class DownloadRepository extends AbstractRepository
$query->logicalAnd(
[
$query->equals('month', $month),
$query->equals('extensionKey', $extensionKey)
$query->equals('extensionKey', $extensionKey),
$query->equals('source', $source)
]
)
);
......
extensions/ter_fe2/Classes/Utility/VersionUtility.php
View file @ 4cda7598
......@@ -31,6 +31,7 @@ class VersionUtility
{
const DOWNLOAD_SOURCE_TER = 1;
const DOWNLOAD_SOURCE_PACKAGIST = 2;
const DOWNLOAD_SOURCE_EM = 3;
/**
* Returns the three part version number (string) from an integer, eg 4012003 -> '4.12.3'
......@@ -205,7 +206,6 @@ class VersionUtility
{
$objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$persistenceManager = $objectManager->get(PersistenceManager::class);
$extensionRepository = $objectManager->get(ExtensionRepository::class);
$month = date('Ym');
$downloadRepository = $objectManager->get(DownloadRepository::class);
......@@ -222,16 +222,14 @@ class VersionUtility
$downloadInfo->setCounter($counter);
$downloadRepository->add($downloadInfo);
} elseif ($downloadInfo->getCounter() > 0) {
$downloadInfo->setCounter($downloadInfo->getCounter() + 1);
// If source = PACKAGIST we set the Packagist data direct no count.
if ($source === self::DOWNLOAD_SOURCE_PACKAGIST) {
$downloadInfo->setCounter($counter);
} else {
$downloadInfo->setCounter($downloadInfo->getCounter() + 1);
}
$downloadRepository->update($downloadInfo);
}
// Sets Extension Last Download Sync to secure oldest syncs happens first.
/** @var Extension $extension */
$extension = $extensionRepository->findOneByExtKey($extensionKey);
$extension->setLastDownloadSync(time());
$extensionRepository->update($extension);
$persistenceManager->persistAll();
}
}
extensions/ter_fe2/Configuration/Commands.php
View file @ 4cda7598
......@@ -38,4 +38,7 @@ return [
'ter:checkForExpiredExtensions' => [
'class' => \T3o\TerFe2\Command\CheckForExpiredExtensions::class
],
'packagist:fetchdownloaddata' => [
'class' => \T3o\TerFe2\Command\PackagistCommand::class
],
];
extensions/ter_fe2/Configuration/TCA/tx_terfe2_domain_model_download.php
View file @ 4cda7598
......@@ -4,7 +4,7 @@ return [
'title' => 'domain_model_download',
'label' => 'extension_key',
'tstamp' => 'tstamp',
'iconfile' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extRelPath('ter_fe2') . 'Resources/Public/Icons/download.png',
'iconfile' => 'EXT:ter_fe2/Resources/Public/Icons/download.png',
],
'interface' => [
'showRecordFieldList' => 'extension_key,version_id',
......
extensions/ter_fe2/Configuration/TCA/tx_terfe2_domain_model_extension.php
View file @ 4cda7598
......@@ -91,6 +91,7 @@ return [
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_extension.tags',
'config' => [
'type' => 'select',
'renderType' => 'selectSingleBox',
'size' => 3,
'minitems' => 0,
'maxitems' => 9999,
......
extensions/ter_fe2/Resources/Private/Language/locallang.xlf
View file @ 4cda7598
......@@ -459,6 +459,9 @@
<trans-unit id="downloads" xml:space="preserve">
<source>Downloads</source>
</trans-unit>
<trans-unit id="downloads.description" xml:space="preserve">
<source>The total number of downloads done since November 1. 2020 from Extension Manager, TER or Packagist.</source>
</trans-unit>
<trans-unit id="details" xml:space="preserve">
<source>Details</source>
</trans-unit>
......
extensions/ter_fe2/Resources/Private/Language/locallang_db.xlf
View file @ 4cda7598
......@@ -273,8 +273,6 @@
<trans-unit id="tx_terfe2_task_checkforexpiredextensions.description" xml:space="preserve">
<source>Check for expired extensions, if extension is expired a mail to the owner is sent.</source>
</trans-unit>
<<<<<<< HEAD
=======
<trans-unit id="tx_terfe2_task_fetchPackagistDownloadData.name" xml:space="preserve">
<source>[TER FE2] Fetch download data from Packagist</source>
</trans-unit>
......@@ -290,7 +288,6 @@
<trans-unit id="tx_terfe2_provider_soapprovider.name" xml:space="preserve">
<source>SOAP Server</source>
</trans-unit>
>>>>>>> Switch model for downloads and fetch data from packagist
<trans-unit id="tt_content.flexform_pi1.s_def" xml:space="preserve">
<source>General</source>
</trans-unit>
......
extensions/ter_fe2/Resources/Private/Partials/ExtensionSingleInfo.html
View file @ 4cda7598
......@@ -29,10 +29,10 @@
<f:format.date format="%d. %b %Y">{extension.dateOfFirstUpload}</f:format.date>
</dd>
<dt>
<f:translate key="downloads" />
<f:translate key="downloads" /> <i class="fas fa-info-circle" title="<f:translate key='downloads.description' />"></i>
</dt>
<dd>
<f:format.number decimals="0" thousandsSeparator="," decimalSeparator=".">{extension.downloads}</f:format.number>
<f:format.number decimals="0" thousandsSeparator="," decimalSeparator=".">{extensionDownloads}</f:format.number>
</dd>
<dt>
<f:translate key="category" />
......
extensions/ter_fe2/Resources/Private/Templates/Extension/Show.html
View file @ 4cda7598
......@@ -173,7 +173,7 @@
<f:render partial="ExternalButtonLink" arguments="{externalLink: 'https://crowdin.com/project/{extension.crowdinKey}/invite', icon: 'fa-globe', label: 'Crowdin translations'}"/>
<div class="alert alert-secondary">
<f:render partial="ExtensionSingleInfo" arguments="{extension:extension, settings:settings, owner:owner, flattrUrl:flattrUrl, documentationLink:documentationLink, qualityLinkNotBroken:qualityLinkNotBroken, urlToQualityServer:urlToQualityServer}" />
<f:render partial="ExtensionSingleInfo" arguments="{extension:extension, settings:settings, owner:owner, flattrUrl:flattrUrl, documentationLink:documentationLink, qualityLinkNotBroken:qualityLinkNotBroken, urlToQualityServer:urlToQualityServer, extensionDownloads:extensionDownloads}" />
</div>
</div>
</div>
......
extensions/ter_fe2/Resources/Public/Javascript/Default.js
View file @ 4cda7598
......@@ -12,29 +12,47 @@ jQuery(document).ready(function ($) {
type: 'column'
},
title: {
text: 'Downloads by version'
text: 'Downloads by months'
},
xAxis: [
{categories: versionChartData.versions},
{categories: versionChartData.release, linkedTo: 0}
{categories: versionChartData.release}
],
yAxis: {
title: {
text: 'Downloads'
}
},
stackLabels: {
enabled: true,
style: {
fontWeight: 'bold',
color: 'gray'
}
},
},
plotOptions: {
line: {
column: {
stacking: 'normal',
dataLabels: {
enabled: true
},
enableMouseTracking: false
}
},
series: [ {
name: 'Downloads',
data: versionChartData.downloads
} ]
series: [
{
name: 'TER',
data: versionChartData.downloads_ter
},
{
name: 'Packagist',
data: versionChartData.downloads_packagist
},
{
name: 'Extension Manager',
data: versionChartData.downloads_em
}
]
});
}
......
extensions/ter_fe2/ext_localconf.php
View file @ 4cda7598
......@@ -55,4 +55,4 @@ $GLOBALS['TYPO3_CONF_VARS']['SYS']['routing']['aspects']['VersionMapper'] = \T3o
$GLOBALS['TYPO3_CONF_VARS']['MAIL']['templateRootPaths'][789] = 'EXT:ter_fe2/Resources/Private/Templates/Mail';
$GLOBALS['TYPO3_CONF_VARS']['MAIL']['layoutRootPaths'][789] = 'EXT:ter_fe2/Resources/Private/Layouts';
$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['extbase']['commandControllers']['tx_ter_fe2_packagist'] =
\T3o\TerFe2\Command\PackagistCommandController::class;
\T3o\TerFe2\Command\PackagistCommand::class;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment