Switch model for downloads and fetch data from packagist

extensions/ter_fe2/Classes/Command/PackagistCommandController.php

+<?php
+declare(strict_types=1);
+namespace T3o\TerFe2\Command;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use T3o\TerFe2\Domain\Model\Extension;
+use T3o\TerFe2\Domain\Repository\ExtensionRepository;
+use T3o\TerFe2\Domain\Repository\VersionRepository;
+use T3o\TerFe2\Utility\VersionUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
+use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
+
+class PackagistCommandController extends CommandController
+{
+    /**
+     * @var ExtensionRepository
+     */
+    protected $extensionRepository;
+
+    /**
+     * @var VersionRepository
+     */
+    protected $versionRepository;
+
+    /**
+     * @param ExtensionRepository $extensionRepository
+     */
+    public function injectExtensionRepository(ExtensionRepository $extensionRepository)
+    {
+        $this->extensionRepository = $extensionRepository;
+    }
+
+    public function injectVersionRepository(VersionRepository $versionRepository)
+    {
+        $this->versionRepository = $versionRepository;
+    }
+
+
+    /**
+     * Fetch download data from Packagist
+     *
+     * Examples:
+     *
+     * --- Fetch download data from packagist
+     * $ typo3cms packagist:fetchdownloaddata 10
+     *
+     * @param int $limit You can set a limit, default is 10
+     *
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
+     */
+    public function fetchDownloadDataCommand(int $limit = 10)
+    {
+
+        $querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
+        $querySettings->setRespectStoragePage(false);
+        $this->extensionRepository->setDefaultQuerySettings($querySettings);
+        $this->versionRepository->setDefaultQuerySettings($querySettings);
+
+        $extensions = $this->extensionRepository->findWithComposerName($limit);
+
+        // User-Agent https://twitter.com/seldaek/status/1095420243315511297
+        $options  = array('http' => array('user_agent' => 'Thanks a bunch! Fetching packagist Download Data for https://extensions.typo3.org - https://typo3.org/community/teams/typo3org/'));
+        $context  = stream_context_create($options);
+
+        /** @var Extension $extension */
+        foreach ($extensions as $extension) {
+            $this->outputLine('<info> Processing: ' . $extension->getExtKey() . '</info>');
+            $packagistUrl = 'https://packagist.org/packages/' . $extension->getComposerName() . '/downloads.json';
+            /** @var Extension $extension */
+            $extension = $this->extensionRepository->findOneByExtKey($extension->getExtKey());
+            $downloadData = json_decode(file_get_contents($packagistUrl, false, $context), true);
+
+            foreach ($downloadData['package']['downloads']['versions'] as $version => $downloads) {
+
+                $versionFound = $this->versionRepository->findOneByExtensionAndVersionString($extension, $version);
+                if (null === $versionFound) {
+                    continue;
+                }
+
+                if (!$version) {
+                    continue;
+                }
+
+                VersionUtility::updateVersionCounter(
+                    $extension->getExtKey(),
+                    $versionFound->getUid(),
+                    VersionUtility::DOWNLOAD_SOURCE_PACKAGIST,
+                    $downloads['monthly']
+                );
+            }
+        }
+    }
+}
\ No newline at end of file

extensions/ter_fe2/Classes/Controller/ExtensionController.php

@@ -20,6 +20,7 @@ use T3o\Ter\Api\ExtensionVersion;
 use T3o\TerFe2\Domain\Model\Extension;
 use T3o\TerFe2\Provider\FileProvider;
 use T3o\TerFe2\Utility\ExtensionUtility;
+use T3o\TerFe2\Utility\VersionUtility;
 use T3o\TerFe2\Validation\Validator\ComposerNameValidator;
 use TYPO3\CMS\Core\Page\PageRenderer;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -417,12 +418,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
         if (!empty($this->settings['countDownloads'])) {
             $extensionKey = $extension->getExtKey();
             $downloads = $this->session->get('downloads');
-            $versionRepository = $this->objectManager->get(\T3o\TerFe2\Domain\Repository\VersionRepository::class);
             if (empty($downloads) || !in_array($extensionKey, $downloads)) {
-                // Add +1 to download counter and save immediately
-                $version->incrementDownloadCounter();
-                $versionRepository->update($version);
-                $this->persistenceManager->persistAll();
+                VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), VersionUtility::DOWNLOAD_SOURCE_TER);
 
                 // Add extension key to session
                 $downloads[] = $extensionKey;

extensions/ter_fe2/Classes/Domain/Model/Download.php

+<?php
+declare(strict_types=1);
+namespace T3o\TerFe2\Domain\Model;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Extbase\DomainObject\AbstractEntity;
+
+class Download extends AbstractEntity
+{
+
+    /**
+     * @var string
+     */
+    protected $extensionKey = '';
+
+    /**
+     * @var int
+     */
+    protected $versionId = 0;
+
+    /**
+     * @var int
+     */
+    protected $month = 0;
+
+    /**
+     * @var int
+     */
+    protected $counter = 0;
+
+    /**
+     * @var int
+     */
+    protected $source = 0;
+
+    /**
+     * @return string
+     */
+    public function getExtensionKey(): string
+    {
+        return $this->extensionKey;
+    }
+
+    /**
+     * @param string $extensionKey
+     */
+    public function setExtensionKey(string $extensionKey)
+    {
+        $this->extensionKey = $extensionKey;
+    }
+
+    /**
+     * @return int
+     */
+    public function getVersionId(): int
+    {
+        return $this->versionId;
+    }
+
+    /**
+     * @param int $versionId
+     */
+    public function setVersionId(int $versionId)
+    {
+        $this->versionId = $versionId;
+    }
+
+    /**
+     * @return int
+     */
+    public function getMonth(): int
+    {
+        return $this->month;
+    }
+
+    /**
+     * @param int $month
+     */
+    public function setMonth(int $month)
+    {
+        $this->month = $month;
+    }
+
+    /**
+     * @return int
+     */
+    public function getCounter(): int
+    {
+        return $this->counter;
+    }
+
+    /**
+     * @param int $counter
+     */
+    public function setCounter(int $counter)
+    {
+        $this->counter = $counter;
+    }
+
+    /**
+     * @return int
+     */
+    public function getSource(): int
+    {
+        return $this->source;
+    }
+
+    /**
+     * @param int $source
+     */
+    public function setSource(int $source)
+    {
+        $this->source = $source;
+    }
+
+
+}
\ No newline at end of file

extensions/ter_fe2/Classes/Domain/Model/Extension.php

@@ -17,6 +17,7 @@ namespace T3o\TerFe2\Domain\Model;
 use T3o\TerFe2\Utility\VersionUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
+use T3o\TerFe2\Domain\Repository\DownloadRepository;
 use TYPO3\CMS\Extbase\Persistence\ObjectStorage;
 
 /**
@@ -144,6 +145,21 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
      */
     protected $crowdinKey = '';
 
+    /**
+     * @var int
+     */
+    protected $lastDownloadSync;
+
+    /**
+     * @var DownloadRepository
+     */
+    protected $downloadRepository;
+
+    public function injectDownloadRepository(DownloadRepository $downloadRepository)
+    {
+        $this->downloadRepository = $downloadRepository;
+    }
+
     /**
      * Initialize all ObjectStorage instances.
      */
@@ -567,6 +583,30 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
         return json_encode($versions);
     }
 
+    /**
+     * @return string
+     */
+    public function getDownloadsByTimeIntervalAsJson(): string
+    {
+        if(empty($this->versions)) {
+            return '';
+        }
+
+        $intervals = [];
+
+        // The for loop is turned around to ensure the sorting order in the multi dimensional array.
+        for ($i = 5; $i >= 0 ; $i--) {
+            $time = strtotime("-" . $i . "month");
+            $month = date('Ym', $time);
+            $downloads = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyTotal($month, $this->getExtKey());
+            $intervals['release'][] = $month;
+            $intervals['downloads'][] = $downloads;
+            $intervals['versions'][] = '';
+        }
+
+        return json_encode($intervals);
+    }
+
     /**
      * @return \DateTime|null
      */
@@ -735,4 +775,20 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
 
         return array_merge($supportOlderVersions, $supportELTSVersions, $supportDevVersion);
     }
+
+    /**
+     * @return int
+     */
+    public function getLastDownloadSync(): int
+    {
+        return $this->lastDownloadSync;
+    }
+
+    /**
+     * @param int $lastDownloadSync
+     */
+    public function setLastDownloadSync(int $lastDownloadSync)
+    {
+        $this->lastDownloadSync = $lastDownloadSync;
+    }
 }

extensions/ter_fe2/Classes/Domain/Repository/DownloadRepository.php

+<?php
+declare(strict_types=1);
+namespace T3o\TerFe2\Domain\Repository;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use T3o\TerFe2\Domain\Model\Download;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
+use TYPO3\CMS\Core\Utility\ClassNamingUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
+
+class DownloadRepository extends AbstractRepository
+{
+
+    /**
+     * @var string
+     */
+    protected $tableName = 'tx_terfe2_domain_model_download';
+
+    /**
+     * @var QueryBuilder
+     */
+    protected $queryBuilder = QueryBuilder::class;
+    /**
+     * QueueRepository constructor.
+     */
+    public function __construct()
+    {
+        $this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
+        $this->objectManager = GeneralUtility::makeInstance(ObjectManager::class);
+        $this->objectType = ClassNamingUtility::translateRepositoryNameToModelName($this->getRepositoryClassName());
+    }
+
+    /**
+     * @param $month
+     * @param $extensionKey
+     * @param $versionId
+     * @param $source
+     *
+     * @return object|null
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
+     */
+    public function findDownloadCounterByMonthAndExtensionKey($month, $extensionKey, $versionId, $source)
+    {
+
+        $query = $this->createQuery();
+        $query->matching(
+            $query->logicalAnd(
+                [
+                    $query->equals('month', $month),
+                    $query->equals('extensionKey', $extensionKey),
+                    $query->equals('versionId', $versionId),
+                    $query->equals('source', $source),
+
+                ]
+            )
+        );
+
+        return $query->execute()->getFirst();
+    }
+
+    /**
+     * @param $month
+     * @param $extensionKey
+     *
+     * @return int
+     */
+    public function findDownloadCounterByMonthAndExtensionKeyTotal($month, $extensionKey)
+    {
+        $total = 0;
+
+        $querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
+        $querySettings->setRespectStoragePage(false);
+
+        $query = $this->createQuery();
+        $query->setQuerySettings($querySettings);
+
+        $query->matching(
+            $query->logicalAnd(
+                [
+                    $query->equals('month', $month),
+                    $query->equals('extensionKey', $extensionKey)
+                ]
+            )
+        );
+
+        $downloads = $query->execute()->toArray();
+
+        /** @var Download $download */
+        foreach ($downloads as $download) {
+            $total = $total + $download->getCounter();
+        }
+
+        return $total;
+    }
+
+}
\ No newline at end of file

extensions/ter_fe2/Classes/Domain/Repository/ExtensionRepository.php

@@ -16,6 +16,8 @@ namespace T3o\TerFe2\Domain\Repository;
 
 use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
+use TYPO3\CMS\Extbase\Persistence\QueryInterface;
 
 /**
  * Repository for \T3o\TerFe2\Domain\Model\Extension
@@ -218,4 +220,26 @@ class ExtensionRepository extends \T3o\TerFe2\Domain\Repository\AbstractReposito
             time()
         );
     }
+
+    /*
+     * @param int $limit
+     * @return array|\TYPO3\CMS\Extbase\Persistence\QueryResultInterface
+     */
+    public function findWithComposerName(int $limit = 10)
+    {
+        $querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
+        $querySettings->setRespectStoragePage(false);
+
+        $query = $this->createQuery();
+        $query->setQuerySettings($querySettings);
+        $query->matching(
+            $query->logicalNot(
+                    $query->equals('composer_name', '')
+            )
+        );
+        $query->setOrderings(['lastDownloadSync' => QueryInterface::ORDER_ASCENDING]);
+        $query->setLimit($limit);
+
+        return $query->execute();
+    }
 }

extensions/ter_fe2/Classes/Utility/VersionUtility.php

@@ -15,13 +15,23 @@ namespace T3o\TerFe2\Utility;
  */
 
 use T3o\TerFe2\Service\LTSVersionService;
+use T3o\TerFe2\Domain\Model\Download;
+use T3o\TerFe2\Domain\Model\Extension;
+use T3o\TerFe2\Domain\Repository\DownloadRepository;
+use T3o\TerFe2\Domain\Repository\ExtensionRepository;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
 
 /**
  * Utilities to manage versions
  */
 class VersionUtility
 {
+
+    const DOWNLOAD_SOURCE_TER = 1;
+    const DOWNLOAD_SOURCE_PACKAGIST = 2;
+
     /**
      * Returns the three part version number (string) from an integer, eg 4012003 -> '4.12.3'
      *
@@ -180,4 +190,49 @@ class VersionUtility
         $parts = explode('.', $versionString) ?: [];
         return (int)array_shift($parts);
     }
+    
+    /**
+     * @param String $extensionKey
+     * @param int $versionId
+     * @param int $source
+     * @param int $counter
+     *
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
+     * @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
+     */
+    public static function updateVersionCounter(String $extensionKey, int $versionId, int $source = self::DOWNLOAD_SOURCE_TER, int $counter = 1)
+    {
+        $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
+        $persistenceManager = $objectManager->get(PersistenceManager::class);
+        $extensionRepository = $objectManager->get(ExtensionRepository::class);
+
+
+        $month = date('Ym');
+        $downloadRepository = $objectManager->get(DownloadRepository::class);
+
+        /** @var Download $downloadInfo */
+        $downloadInfo = $downloadRepository->findDownloadCounterByMonthAndExtensionKey($month,$extensionKey,$versionId, $source);
+
+        if (null === $downloadInfo) {
+            $downloadInfo = new Download();
+            $downloadInfo->setExtensionKey($extensionKey);
+            $downloadInfo->setSource($source);
+            $downloadInfo->setMonth($month);
+            $downloadInfo->setVersionId($versionId);
+            $downloadInfo->setCounter($counter);
+            $downloadRepository->add($downloadInfo);
+        } elseif ($downloadInfo->getCounter() > 0) {
+            $downloadInfo->setCounter($downloadInfo->getCounter()+1);
+            $downloadRepository->update($downloadInfo);
+        }
+
+        // Sets Extension Last Download Sync to secure oldest syncs happens first.
+        /** @var Extension $extension */
+        $extension = $extensionRepository->findOneByExtKey($extensionKey);
+        $extension->setLastDownloadSync(time());
+        $extensionRepository->update($extension);
+
+        $persistenceManager->persistAll();
+    }
 }

extensions/ter_fe2/Configuration/TCA/tx_terfe2_domain_model_download.php

+<?php
+return [
+    'ctrl' => [
+        'title' => 'domain_model_download',
+        'label' => 'extension_key',
+        'tstamp' => 'tstamp',
+        'iconfile' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extRelPath('ter_fe2') . 'Resources/Public/Icons/tag.gif',
+    ],
+    'interface' => [
+        'showRecordFieldList' => 'extension_key,version_id',
+    ],
+    'types' => [
+        '1' => ['showitem' => 'extension_key'],
+    ],
+    'palettes' => [
+        '1' => ['showitem' => ''],
+    ],
+    'columns' => [
+        'extension_key' => [
+            'label' => 'Extension Key',
+            'config' => [
+                'type' => 'input',
+            ],
+        ],
+        'version_id' => [
+            'label' => 'Version ID',
+            'config' => [
+                'type' => 'input',
+            ],
+        ],
+        'source' => [
+            'label' => 'Download Source',