[BUGFIX] Improve performance of extension scanner

This patch adds a request queue for the massive amount of AJAX calls
in the extension scanner. This patch mitigates the problem of massive
server load in case the amount of files to be scanned is very high.

This patch mitigates the problem by allowing a maximum of 10 concurrent
requests. After the release of TYPO3 v9 LTS, this scanner module should
be refactored further.

Resolves: #86436
Releases: master
Change-Id: I824dfb74aae46d6b00690e7ca6553305c4221f0f
Reviewed-on: https://review.typo3.org/58510
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
Reviewed-by: Stefan Neufeind <typo3.neufeind@speedpartner.de>
Tested-by: Stefan Neufeind <typo3.neufeind@speedpartner.de>

diff --git a/typo3/sysext/install/Resources/Public/JavaScript/Modules/ExtensionScanner.js b/typo3/sysext/install/Resources/Public/JavaScript/Modules/ExtensionScanner.js
index 88480db..0e60536 100644 (file)
--- a/typo3/sysext/install/Resources/Public/JavaScript/Modules/ExtensionScanner.js
+++ b/typo3/sysext/install/Resources/Public/JavaScript/Modules/ExtensionScanner.js
@@ -27,6 +27,10 @@ define(['jquery',
     selectorExtensionContainer: '.t3js-extensionScanner-extension',
     selectorNumberOfFiles: '.t3js-extensionScanner-number-of-files',
     selectorScanSingleTrigger: '.t3js-extensionScanner-scan-single',
+    ajaxRequests: 0,
+    ajaxQueue: [],
+    ajaxActive: 0,
+    ajaxMaxConcurrent: 10,
 
     initialize: function(currentModal) {
       var self = this;
@@ -55,22 +59,53 @@ define(['jquery',
       });
     },
 
+    addAjaxCallToQueue: function(obj) {
+      this.ajaxRequests++;
+      var oldSuccess = obj.success;
+      var oldError = obj.error;
+      var that = this;
+      var callback = function() {
+        that.ajaxRequests--;
+        if (that.ajaxActive === that.ajaxMaxConcurrent) {
+          $.ajax(that.ajaxQueue.shift());
+        } else {
+          that.ajaxActive--;
+        }
+      };
+      obj.success = function(resp, xhr, status) {
+        callback();
+        if (oldSuccess) oldSuccess(resp, xhr, status);
+      };
+      obj.error = function(xhr, status, error) {
+        callback();
+        if (oldError) oldError(xhr, status, error);
+      };
+      if (this.ajaxActive === this.ajaxMaxConcurrent) {
+        this.ajaxQueue.push(obj);
+      } else {
+        this.ajaxActive++;
+        $.ajax(obj);
+      }
+    },
+
     getData: function() {
       var modalContent = this.currentModal.find(this.selectorModalBody);
-      $.ajax({
-        url: Router.getUrl('extensionScannerGetData'),
-        cache: false,
-        success: function(data) {
-          if (data.success === true) {
-            modalContent.empty().append(data.html);
-          } else {
-            Notification.error('Something went wrong');
+      this.addAjaxCallToQueue(
+        {
+          url: Router.getUrl('extensionScannerGetData'),
+          cache: false,
+          success: function(data) {
+            if (data.success === true) {
+              modalContent.empty().append(data.html);
+            } else {
+              Notification.error('Something went wrong');
+            }
+          },
+          error: function(xhr) {
+            Router.handleAjaxError(xhr, modalContent);
           }
-        },
-        error: function(xhr) {
-          Router.handleAjaxError(xhr, modalContent);
         }
-      });
+      );
     },
 
     /**
@@ -149,26 +184,28 @@ define(['jquery',
 
       if (numberOfScannedExtensions === numberOfExtensions) {
         Notification.success('Scan finished', 'All extensions have been scanned');
-        $.ajax({
-          url: Router.getUrl(),
-          method: 'POST',
-          data: {
-            'install': {
-              'action': 'extensionScannerMarkFullyScannedRestFiles',
-              'token': self.currentModal.find(self.selectorModuleContent).data('extension-scanner-mark-fully-scanned-rest-files-token'),
-              'hashes': self.uniqueArray(this.listOfAffectedRestFileHashes)
-            }
-          },
-          cache: false,
-          success: function(data) {
-            if (data.success === true) {
-              Notification.success('Marked not affected files', 'Marked ' + data.markedAsNotAffected + ' ReST files as not affected.');
+        this.addAjaxCallToQueue(
+          {
+            url: Router.getUrl(),
+            method: 'POST',
+            data: {
+              'install': {
+                'action': 'extensionScannerMarkFullyScannedRestFiles',
+                'token': self.currentModal.find(self.selectorModuleContent).data('extension-scanner-mark-fully-scanned-rest-files-token'),
+                'hashes': self.uniqueArray(this.listOfAffectedRestFileHashes)
+              }
+            },
+            cache: false,
+            success: function(data) {
+              if (data.success === true) {
+                Notification.success('Marked not affected files', 'Marked ' + data.markedAsNotAffected + ' ReST files as not affected.');
+              }
+            },
+            error: function(xhr) {
+              Router.handleAjaxError(xhr, modalContent);
             }
-          },
-          error: function(xhr) {
-            Router.handleAjaxError(xhr, modalContent);
           }
-        });
+        );
       }
     },
 
@@ -204,132 +241,137 @@ define(['jquery',
       $extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-files').empty().text('0');
       $extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-lines').empty().text('0');
       this.setProgressForAll();
-      $.ajax({
-        url: Router.getUrl(),
-        method: 'POST',
-        data: {
-          'install': {
-            'action': 'extensionScannerFiles',
-            'token': executeToken,
-            'extension': extension
-          }
-        },
-        cache: false,
-        success: function(data) {
-          if (data.success === true && Array.isArray(data.files)) {
-            var numberOfFiles = data.files.length;
-            if (numberOfFiles > 0) {
-              self.setStatusMessageForScan(extension, 0, numberOfFiles);
-              $extensionContainer.find('.t3js-extensionScanner-extension-body').text('');
-              var doneFiles = 0;
-              data.files.forEach(function(file) {
-                $.ajax({
-                  method: 'POST',
-                  data: {
-                    'install': {
-                      'action': 'extensionScannerScanFile',
-                      'token': self.currentModal.find(self.selectorModuleContent).data('extension-scanner-scan-file-token'),
-                      'extension': extension,
-                      'file': file
-                    }
-                  },
-                  url: Router.getUrl(),
-                  cache: false,
-                  success: function(fileData) {
-                    doneFiles = doneFiles + 1;
-                    self.setStatusMessageForScan(extension, doneFiles, numberOfFiles);
-                    self.setProgressForScan(extension, doneFiles, numberOfFiles);
-                    if (fileData.success && $.isArray(fileData.matches)) {
-                      $(fileData.matches).each(function() {
-                        hitFound = true;
-                        var match = this;
-                        var aMatch = modalContent.find(hitTemplate).clone();
-                        aMatch.find('.t3js-extensionScanner-hit-file-panel-head').attr('href', '#collapse' + match.uniqueId);
-                        aMatch.find('.t3js-extensionScanner-hit-file-panel-body').attr('id', 'collapse' + match.uniqueId);
-                        aMatch.find('.t3js-extensionScanner-hit-filename').text(file);
-                        aMatch.find('.t3js-extensionScanner-hit-message').text(match.message);
-                        if (match.indicator === 'strong') {
-                          aMatch.find('.t3js-extensionScanner-hit-file-panel-head .badges')
-                            .append('<span class="badge" title="Reliable match, false positive unlikely">strong</span>');
-                        } else {
-                          aMatch.find('.t3js-extensionScanner-hit-file-panel-head .badges')
-                            .append('<span class="badge" title="Probable match, but can be a false positive">weak</span>');
-                        }
-                        if (match.silenced === true) {
-                          aMatch.find('.t3js-extensionScanner-hit-file-panel-head .badges')
-                            .append('<span class="badge" title="Match has been annotated by extension author as false positive match">silenced</span>');
+      var that = this;
+      this.addAjaxCallToQueue(
+        {
+          url: Router.getUrl(),
+          method: 'POST',
+          data: {
+            'install': {
+              'action': 'extensionScannerFiles',
+              'token': executeToken,
+              'extension': extension
+            }
+          },
+          cache: false,
+          success: function(data) {
+            if (data.success === true && Array.isArray(data.files)) {
+              var numberOfFiles = data.files.length;
+              if (numberOfFiles > 0) {
+                self.setStatusMessageForScan(extension, 0, numberOfFiles);
+                $extensionContainer.find('.t3js-extensionScanner-extension-body').text('');
+                var doneFiles = 0;
+                data.files.forEach(function(file) {
+                  that.addAjaxCallToQueue(
+                    {
+                      method: 'POST',
+                      data: {
+                        'install': {
+                          'action': 'extensionScannerScanFile',
+                          'token': self.currentModal.find(self.selectorModuleContent).data('extension-scanner-scan-file-token'),
+                          'extension': extension,
+                          'file': file
                         }
-                        aMatch.find('.t3js-extensionScanner-hit-file-lineContent').empty().text(match.lineContent);
-                        aMatch.find('.t3js-extensionScanner-hit-file-line').empty().text(match.line + ': ');
-                        if ($.isArray(match.restFiles)) {
-                          $(match.restFiles).each(function() {
-                            var restFile = this;
-                            var aRest = modalContent.find(restTemplate).clone();
-                            aRest.find('.t3js-extensionScanner-hit-rest-panel-head').attr('href', '#collapse' + restFile.uniqueId);
-                            aRest.find('.t3js-extensionScanner-hit-rest-panel-head .badge').empty().text(restFile.version);
-                            aRest.find('.t3js-extensionScanner-hit-rest-panel-body').attr('id', 'collapse' + restFile.uniqueId);
-                            aRest.find('.t3js-extensionScanner-hit-rest-headline').text(restFile.headline);
-                            aRest.find('.t3js-extensionScanner-hit-rest-body').text(restFile.content);
-                            aRest.addClass('panel-' + restFile.class);
-                            aMatch.find('.t3js-extensionScanner-hit-file-rest-container').append(aRest);
-                            self.listOfAffectedRestFileHashes.push(restFile.file_hash);
+                      },
+                      url: Router.getUrl(),
+                      cache: false,
+                      success: function(fileData) {
+                        doneFiles = doneFiles + 1;
+                        self.setStatusMessageForScan(extension, doneFiles, numberOfFiles);
+                        self.setProgressForScan(extension, doneFiles, numberOfFiles);
+                        if (fileData.success && $.isArray(fileData.matches)) {
+                          $(fileData.matches).each(function() {
+                            hitFound = true;
+                            var match = this;
+                            var aMatch = modalContent.find(hitTemplate).clone();
+                            aMatch.find('.t3js-extensionScanner-hit-file-panel-head').attr('href', '#collapse' + match.uniqueId);
+                            aMatch.find('.t3js-extensionScanner-hit-file-panel-body').attr('id', 'collapse' + match.uniqueId);
+                            aMatch.find('.t3js-extensionScanner-hit-filename').text(file);
+                            aMatch.find('.t3js-extensionScanner-hit-message').text(match.message);
+                            if (match.indicator === 'strong') {
+                              aMatch.find('.t3js-extensionScanner-hit-file-panel-head .badges')
+                                .append('<span class="badge" title="Reliable match, false positive unlikely">strong</span>');
+                            } else {
+                              aMatch.find('.t3js-extensionScanner-hit-file-panel-head .badges')
+                                .append('<span class="badge" title="Probable match, but can be a false positive">weak</span>');
+                            }
+                            if (match.silenced === true) {
+                              aMatch.find('.t3js-extensionScanner-hit-file-panel-head .badges')
+                                .append('<span class="badge" title="Match has been annotated by extension author as false positive match">silenced</span>');
+                            }
+                            aMatch.find('.t3js-extensionScanner-hit-file-lineContent').empty().text(match.lineContent);
+                            aMatch.find('.t3js-extensionScanner-hit-file-line').empty().text(match.line + ': ');
+                            if ($.isArray(match.restFiles)) {
+                              $(match.restFiles).each(function() {
+                                var restFile = this;
+                                var aRest = modalContent.find(restTemplate).clone();
+                                aRest.find('.t3js-extensionScanner-hit-rest-panel-head').attr('href', '#collapse' + restFile.uniqueId);
+                                aRest.find('.t3js-extensionScanner-hit-rest-panel-head .badge').empty().text(restFile.version);
+                                aRest.find('.t3js-extensionScanner-hit-rest-panel-body').attr('id', 'collapse' + restFile.uniqueId);
+                                aRest.find('.t3js-extensionScanner-hit-rest-headline').text(restFile.headline);
+                                aRest.find('.t3js-extensionScanner-hit-rest-body').text(restFile.content);
+                                aRest.addClass('panel-' + restFile.class);
+                                aMatch.find('.t3js-extensionScanner-hit-file-rest-container').append(aRest);
+                                self.listOfAffectedRestFileHashes.push(restFile.file_hash);
+                              });
+                            }
+                            var panelClass =
+                              aMatch.find('.panel-breaking', '.t3js-extensionScanner-hit-file-rest-container').length > 0
+                                ? 'panel-danger'
+                                : 'panel-warning';
+                            aMatch.addClass(panelClass);
+                            $extensionContainer.find('.t3js-extensionScanner-extension-body').removeClass('hide').append(aMatch);
+                            if (panelClass === 'panel-danger') {
+                              $extensionContainer.removeClass('panel-warning').addClass(panelClass);
+                            }
+                            if (panelClass === 'panel-warning' && !$extensionContainer.hasClass('panel-danger')) {
+                              $extensionContainer.addClass(panelClass);
+                            }
                           });
                         }
-                        var panelClass =
-                          aMatch.find('.panel-breaking', '.t3js-extensionScanner-hit-file-rest-container').length > 0
-                            ? 'panel-danger'
-                            : 'panel-warning';
-                        aMatch.addClass(panelClass);
-                        $extensionContainer.find('.t3js-extensionScanner-extension-body').removeClass('hide').append(aMatch);
-                        if (panelClass === 'panel-danger') {
-                          $extensionContainer.removeClass('panel-warning').addClass(panelClass);
+                        if (fileData.success) {
+                          var currentLinesOfCode = parseInt($extensionContainer.find('.t3js-extensionScanner-extension-body-loc').text());
+                          $extensionContainer.find('.t3js-extensionScanner-extension-body-loc').empty().text(currentLinesOfCode + parseInt(fileData.effectiveCodeLines));
+                          if (fileData.isFileIgnored) {
+                            var currentIgnoredFiles = parseInt($extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-files').text());
+                            $extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-files').empty().text(currentIgnoredFiles + 1);
+                          }
+                          var currentIgnoredLines = parseInt($extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-lines').text());
+                          $extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-lines').empty().text(currentIgnoredLines + parseInt(fileData.ignoredLines));
                         }
-                        if (panelClass === 'panel-warning' && !$extensionContainer.hasClass('panel-danger')) {
-                          $extensionContainer.addClass(panelClass);
+                        if (doneFiles === numberOfFiles) {
+                          if (!hitFound) {
+                            $extensionContainer.addClass('panel-success');
+                          }
+                          $extensionContainer.addClass('t3js-extensionscan-finished');
+                          self.setProgressForAll();
+                          $extensionContainer.find('.t3js-extensionScanner-scan-single').text('Rescan').attr('disabled', null);
                         }
-                      });
-                    }
-                    if (fileData.success) {
-                      var currentLinesOfCode = parseInt($extensionContainer.find('.t3js-extensionScanner-extension-body-loc').text());
-                      $extensionContainer.find('.t3js-extensionScanner-extension-body-loc').empty().text(currentLinesOfCode + parseInt(fileData.effectiveCodeLines));
-                      if (fileData.isFileIgnored) {
-                        var currentIgnoredFiles = parseInt($extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-files').text());
-                        $extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-files').empty().text(currentIgnoredFiles + 1);
-                      }
-                      var currentIgnoredLines = parseInt($extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-lines').text());
-                      $extensionContainer.find('.t3js-extensionScanner-extension-body-ignored-lines').empty().text(currentIgnoredLines + parseInt(fileData.ignoredLines));
-                    }
-                    if (doneFiles === numberOfFiles) {
-                      if (!hitFound) {
-                        $extensionContainer.addClass('panel-success');
+                      },
+                      error: function(data) {
+                        doneFiles = doneFiles + 1;
+                        self.setStatusMessageForScan(extension, doneFiles, numberOfFiles);
+                        self.setProgressForScan(extension, doneFiles, numberOfFiles);
+                        self.setProgressForAll();
+                        Notification.error('Oops, an error occurred', 'Please look at the console output for details');
+                        console.error(data);
                       }
-                      $extensionContainer.addClass('t3js-extensionscan-finished');
-                      self.setProgressForAll();
-                      $extensionContainer.find('.t3js-extensionScanner-scan-single').text('Rescan').attr('disabled', null);
                     }
-                  },
-                  error: function(data) {
-                    doneFiles = doneFiles + 1;
-                    self.setStatusMessageForScan(extension, doneFiles, numberOfFiles);
-                    self.setProgressForScan(extension, doneFiles, numberOfFiles);
-                    self.setProgressForAll();
-                    Notification.error('Oops, an error occurred', 'Please look at the console output for details');
-                    console.error(data);
-                  }
+                  );
                 });
-              });
+              } else {
+                Notification.warning('No files found', 'The extension EXT:' + extension + ' contains no files we can scan');
+              }
             } else {
-              Notification.warning('No files found', 'The extension EXT:' + extension + ' contains no files we can scan');
+              Notification.error('Oops, an error occurred', 'Please look at the console output for details');
+              console.error(data);
             }
-          } else {
-            Notification.error('Oops, an error occurred', 'Please look at the console output for details');
-            console.error(data);
+          },
+          error: function(xhr) {
+            Router.handleAjaxError(xhr, modalContent);
           }
-        },
-        error: function(xhr) {
-          Router.handleAjaxError(xhr, modalContent);
         }
-      });
+      );
     }
   };
 });