[TASK] Script to update a 3rd party extension 00/41500/2
authorXavier Perseguers <xavier@typo3.org>
Thu, 16 Jul 2015 18:14:43 +0000 (20:14 +0200)
committerXavier Perseguers <xavier@typo3.org>
Thu, 16 Jul 2015 18:16:03 +0000 (20:16 +0200)
Change-Id: I10ee82c4d2dabde90e488fbb1cbc35b6d45e8c8a
Reviewed-on: http://review.typo3.org/41500
Reviewed-by: Xavier Perseguers <xavier@typo3.org>
Tested-by: Xavier Perseguers <xavier@typo3.org>
LEGACY/bin/pootle-upgrade-extension [deleted file]
LEGACY/bin/t3xutils.phar [deleted file]
typo3/files/scripts/bin/build-language-pack
typo3/files/scripts/bin/build-sysext-language-pack
typo3/files/scripts/bin/migration/1-migrate-typo3-xliff
typo3/files/scripts/bin/t3xutils.phar [new file with mode: 0755]
typo3/files/scripts/bin/update-extension [new file with mode: 0755]
typo3/files/scripts/bin/update-typo3-core-template
typo3/tasks/main.yml
typo3/templates/scripts/etc/pootle.conf.j2

diff --git a/LEGACY/bin/pootle-upgrade-extension b/LEGACY/bin/pootle-upgrade-extension
deleted file mode 100755 (executable)
index 57e19d2..0000000
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errtrace
-set -o nounset
-
-#  Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR
-trap onexit 1 2 3 15
-
-#--- onexit() -----------------------------------------------------
-#  @param $1 integer  (optional) Exit status.  If not set, use `$?'
-function onexit() {
-       local exit_status=${1:-$?}
-       if [ $exit_status -gt 0 ]; then
-               echo Exiting $0 with $exit_status
-       fi
-       exit $exit_status
-}
-
-T3XUTILS=/var/www/vhosts/pootle.typo3.org/home/scripts/bin/t3xutils.phar
-POOTLE_MANAGE=/var/www/vhosts/pootle.typo3.org/home/scripts/bin/pootle-manage
-
-# --- check_available_languages() ---------------------------------
-# @param $1 string               Pootle Project Name
-function check_available_languages() {
-       local NEED_UPDATE=0
-       local PROJECT_NAME=$1
-        for LANGUAGE_KEY in $(ls /var/www/vhosts/pootle.typo3.org/home/templates); do
-               local LANGUAGE_DIRECTORY="/var/www/vhosts/pootle.typo3.org/pootle/po/$PROJECT_NAME/$LANGUAGE_KEY"
-                if [ "$PROJECT_NAME" != "" ] && [ -d /var/www/vhosts/pootle.typo3.org/pootle/po/$PROJECT_NAME ] && [ ! -d $LANGUAGE_DIRECTORY ]; then
-                        mkdir -p $LANGUAGE_DIRECTORY
-                        echo "[NOTICE] Add project language \"$LANGUAGE_KEY\""
-                       NEED_UPDATE=1
-                fi
-        done
-       if [ $NEED_UPDATE == 1 ]; then
-               echo "[NOTICE] Update translation project"
-               $POOTLE_MANAGE update_translation_projects --project=$1 1>/dev/null 2>/dev/null
-       fi
-}
-
-SAVEIFS=$IFS
-
-XSL_LLXML2TEMPLATE="/var/www/vhosts/pootle.typo3.org/home/scripts/etc/xsl/llxml2template.xsl"
-TEMP_DIRECTORY="/var/www/vhosts/pootle.typo3.org/home/tmp"
-
-EXTENSION=$1
-VERSION=$2
-
-PROJECT_NAME="TYPO3.TYPO3.ext.$EXTENSION"
-PROJECT_DIRECTORY="/var/www/vhosts/pootle.typo3.org/pootle/po/$PROJECT_NAME"
-#
-# Check if the project exist
-if [ ! -d $PROJECT_DIRECTORY ]; then
-       echo
-       echo "[ERROR] Project $PROJECT_NAME doesn't exist in Pootle, please create it before using this script"
-       echo "[ERROR] Please create directory: $PROJECT_DIRECTORY"
-       onexit 1
-fi
-
-#
-# Check if the templates directory exist
-TEMPLATE_DIRECTORY="$PROJECT_DIRECTORY/templates"
-if [ ! -d $TEMPLATE_DIRECTORY ]; then
-        echo
-        echo "[INFO] Templates directory for project $PROJECT_NAME doesn't exist in Pootle"
-       echo "[NOTICE] Create the templates directory"
-       mkdir -p $TEMPLATE_DIRECTORY
-fi
-
-cd $TEMP_DIRECTORY >/dev/null
-
-echo "[INFO] Downloading $EXTENSION $VERSION from TER"
-
-EXTENSION_T3X=$(echo "${EXTENSION}_${VERSION}.t3x" | tr '[:upper:]' '[:lower:]')
-
-#
-# Download t3x
-rm -f $EXTENSION_T3X
-wget -q http://typo3.org/fileadmin/ter/${EXTENSION_T3X:0:1}/${EXTENSION_T3X:1:1}/$EXTENSION_T3X
-
-#
-# Extract t3x
-#extract-t3x $EXTENSION_T3X $EXTENSION > /dev/null
-$T3XUTILS extract $EXTENSION_T3X $EXTENSION >/dev/null
-
-IFS=$(echo -en "\n\b")
-
-#
-# Create directory structure for templates
-for FILE in $(find $EXTENSION/ -name \*.xml); do
-       grep "T3locallang" "$FILE" 1> /dev/null
-       if [ $? == 0 ]; then
-               mkdir -p $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$(dirname $FILE);
-               cp $FILE $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$FILE;
-       fi
-done
-for FILE in $(find $EXTENSION/ -name \*.xlf); do
-       # Count number of dots in filename to skip e.g., de.locallang.xlf
-       DOTS=$(echo -n $(basename $FILE) | sed 's/[^.]//g' | wc -c)
-       if [ $DOTS -eq 1 ]; then
-               mkdir -p $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$(dirname $FILE);
-               cp $FILE $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$FILE;
-       fi
-done
-
-cd $TEMP_DIRECTORY
-for FILE in $(find TYPO3.TYPO3.ext.$EXTENSION -name \*.xml); do
-       grep "T3locallang" "$FILE" 1> /dev/null
-       if [ $? == 0 ]; then
-               XLF_FILE=$(echo $FILE | sed s/.xml$/.xlf/)
-               if [ ! -f $XLF_FILE ]; then
-                       echo "[INFO] Convert LLXML to XLIFF for file $FILE"
-                       if [ ! -f $XLF_FILE ]; then
-                               xsltproc --stringparam lang en \
-                                       --stringparam extension $EXTENSION \
-                                       --stringparam date "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" $XSL_LLXML2TEMPLATE $FILE | \
-                                       xmlstarlet ed --insert "/xliff/file/body/trans-unit" --type attr -n xml:space -v preserve > $XLF_FILE
-                       fi
-               fi
-       fi
-       rm -f $FILE
-done
-
-#
-# Deploy new templates files
-echo "[INFO] Deploy new templates to $TEMPLATE_DIRECTORY"
-cd $TEMPLATE_DIRECTORY/../../ && \
-       tar cfzp $TEMP_DIRECTORY/backup/$PROJECT_NAME-$(date +%Y%m%d).tar.gz $PROJECT_NAME && \
-       cd - && \
-       rm -Rf $TEMPLATE_DIRECTORY && \
-       mkdir -p $TEMPLATE_DIRECTORY && \
-       cp -a $TEMP_DIRECTORY/$PROJECT_NAME/* $TEMPLATE_DIRECTORY || \
-               onexit 5
-
-#
-# Check project available language
-check_available_languages $PROJECT_NAME
-
-#
-# Update project from templates
-echo "[INFO] Update from templates"
-$POOTLE_MANAGE update_from_templates --project=$PROJECT_NAME 1>/dev/null 2>/dev/null
-
-echo "[INFO] Refresh statistics"
-$POOTLE_MANAGE refresh_stats --project=$PROJECT_NAME 1>/dev/null 2>/dev/null
-
-echo "[NOTICE] Register version $VERSION for EXT:$EXTENSION"
-echo -n $VERSION > $PROJECT_DIRECTORY/.ter-version
-
-#
-# Cleanup
-cd $TEMP_DIRECTORY
-rm -f $EXTENSION_T3X
-if [ -d $EXTENSION ]; then
-       rm -rf $EXTENSION
-       rm -rf TYPO3.TYPO3.ext.$EXTENSION
-fi
-
-IFS=$SAVEIFS
-
-onexit
diff --git a/LEGACY/bin/t3xutils.phar b/LEGACY/bin/t3xutils.phar
deleted file mode 100755 (executable)
index e6b1b30..0000000
Binary files a/LEGACY/bin/t3xutils.phar and /dev/null differ
index 705a912..3dc9a44 100755 (executable)
@@ -18,7 +18,7 @@ cd $(dirname $0)
 #if [ -z "$FORCE_EXTENSION" ]; then
 #      # Sync whole stores, --project seems not totally reliable
 #      echo "Synchronizing all stores ... "
-#      ${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores 2>&1
+#      ${POOTLE_MANAGE} sync_stores 2>&1
 #fi
 
 for EXTNAME in $EXTENSIONS; do
@@ -27,11 +27,11 @@ for EXTNAME in $EXTENSIONS; do
        echo "Packaging $EXTNAME"
        if [ ! -z "$FORCE_EXTENSION" ]; then
                echo -n "   synchronizing store ... "
-               ${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
+               ${POOTLE_MANAGE} sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
                echo "done."
        fi
        echo -n "   updating statistics ... "
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
+       ${POOTLE_MANAGE} refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
        echo "done."
        pushd ${POOTLE_PO}/$EXT_PREFIX.$EXTNAME >/dev/null
        rm -rf ${TER_L10N_PATH}/_$EXTNAME
@@ -79,7 +79,7 @@ for EXTNAME in $EXTENSIONS; do
                esac
 
                echo -n "   processing $LANG ... "
-               #${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
+               #${POOTLE_MANAGE} sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
 
                # Compute current state
                STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time
index fcd26bc..c529b8e 100755 (executable)
@@ -18,7 +18,7 @@ cd $(dirname $0)
 #if [ -z "$FORCE_EXTENSION" ]; then
 #      # Sync whole stores, --project seems not totally reliable
 #      echo "Synchronizing all stores ... "
-#      ${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores 2>&1
+#      ${POOTLE_MANAGE} sync_stores 2>&1
 #fi
 
 for EXTNAME in $EXTENSIONS; do
@@ -27,11 +27,11 @@ for EXTNAME in $EXTENSIONS; do
        echo "Packaging $EXTNAME"
        if [ ! -z "$FORCE_EXTENSION" ]; then
                echo -n "   synchronizing store ... "
-               ${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
+               ${POOTLE_MANAGE} sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
                echo "done."
        fi
        echo -n "   updating statistics ... "
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
+       ${POOTLE_MANAGE} refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
        echo "done."
        pushd ${POOTLE_PO}/$EXT_PREFIX.$EXTNAME >/dev/null
 
@@ -92,7 +92,7 @@ for EXTNAME in $EXTENSIONS; do
                        esac
 
                        echo -n "   processing $LANG ... "
-                       #${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
+                       #${POOTLE_MANAGE} sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
 
                        # Compute current state
                        STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time
index d3e5e3f..e542148 100755 (executable)
@@ -11,7 +11,7 @@ for PROJECT in ${PROJECTS}; do
        pushd ${PROJECT} >/dev/null
 
        echo "[INFO]    Migrating ${PROJECT}"
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --overwrite --project=${PROJECT}
+       ${POOTLE_MANAGE} sync_stores --overwrite --project=${PROJECT}
 
        LANGUAGES=$(find . -maxdepth 1 -type d | cut -b3-)
        for LANGUAGE in ${LANGUAGES}; do
@@ -52,7 +52,7 @@ for PROJECT in ${PROJECTS}; do
        done
 
        echo "[INFO]    Updating Pootle stores for ${PROJECT}"
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_stores --project=${PROJECT}
+       ${POOTLE_MANAGE} update_stores --project=${PROJECT}
 
        popd >/dev/null
 done
diff --git a/typo3/files/scripts/bin/t3xutils.phar b/typo3/files/scripts/bin/t3xutils.phar
new file mode 100755 (executable)
index 0000000..e6b1b30
Binary files /dev/null and b/typo3/files/scripts/bin/t3xutils.phar differ
diff --git a/typo3/files/scripts/bin/update-extension b/typo3/files/scripts/bin/update-extension
new file mode 100755 (executable)
index 0000000..957dfd1
--- /dev/null
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+. /home/pootle/scripts/etc/pootle.conf
+. /home/pootle/scripts/etc/functions
+
+set -o errtrace
+set -o nounset
+
+#  Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR
+trap onexit 1 2 3 15
+
+#--- onexit() -----------------------------------------------------
+#  @param $1 integer  (optional) Exit status.  If not set, use `$?'
+function onexit() {
+       local exit_status=${1:-$?}
+       if [ $exit_status -gt 0 ]; then
+               echo Exiting $0 with $exit_status
+       fi
+       exit $exit_status
+}
+
+if [ $# -lt 2 ]; then
+       echo "Usage: $0 <extension-key> <version>" >&2
+       exit 1
+fi
+
+SAVEIFS=$IFS
+
+EXTENSION=$(echo $1 | tr '[:upper:]' '[:lower:]')
+VERSION=$2
+PROJECT_NAME="TYPO3.ext.${EXTENSION}"
+PROJECT_DIRECTORY="${POOTLE_PO}/${PROJECT_NAME}"
+
+#
+# Check if the project exist
+if [ ! -d ${PROJECT_DIRECTORY} ]; then
+       echo
+       echo "[ERROR]   Project ${PROJECT_NAME} doesn't exist in Pootle, please create it before using this script"
+       echo "[ERROR]   Please create directory: ${PROJECT_DIRECTORY}"
+       onexit 1
+fi
+
+#
+# Check if the templates directory exist
+TEMPLATE_DIRECTORY="${PROJECT_DIRECTORY}/templates"
+if [ ! -d $TEMPLATE_DIRECTORY ]; then
+        echo
+        echo "[INFO]    Templates directory for project ${PROJECT_NAME} doesn't exist in Pootle"
+       echo "[NOTICE]  Create the templates directory"
+       mkdir -p $TEMPLATE_DIRECTORY
+fi
+
+pushd /tmp >/dev/null
+
+echo "[INFO]    Downloading ${EXTENSION} ${VERSION} from TER"
+
+#
+# Download t3x
+EXTENSION_T3X=${EXTENSION}_${VERSION}.t3x
+rm -f ${EXTENSION_T3X}
+${T3XUTILS} fetch ${EXTENSION} ${VERSION}
+if [ ! -f ${EXTENSION_T3X} ]; then
+       echo "[ERROR]   Could not download extension from TER"
+       onexit 1
+fi
+
+#
+# Extract t3x
+${T3XUTILS} extract ${EXTENSION_T3X} ${EXTENSION} >/dev/null
+
+IFS=$(echo -en "\n\b")
+
+#
+# Create directory structure for templates
+for FILE in $(find ${EXTENSION}/ -name \*.xml); do
+       grep "T3locallang" "${FILE}" >/dev/null
+       if [ $? -eq 0 ]; then
+               mkdir -p /tmp/TYPO3.ext.$(dirname ${FILE})
+               cp ${FILE} /tmp/TYPO3.ext.${FILE}
+       fi
+done
+for FILE in $(find ${EXTENSION}/ -name \*.xlf); do
+       # Count number of dots in filename to skip e.g., de.locallang.xlf
+       DOTS=$(echo -n $(basename $FILE) | sed 's/[^.]//g' | wc -c)
+       if [ ${DOTS} -eq 1 ]; then
+               mkdir -p /tmp/TYPO3.ext.$(dirname ${FILE})
+               cp ${FILE} /tmp/TYPO3.ext.${FILE}
+       fi
+done
+
+for FILE in $(find TYPO3.ext.${EXTENSION} -name \*.xml); do
+       grep "T3locallang" "${FILE}" >/dev/null
+       if [ $? -eq 0 ]; then
+               XLF_FILE=$(echo ${FILE} | sed s/.xml$/.xlf/)
+               if [ ! -f ${XLF_FILE} ]; then
+                       echo "[INFO]    Convert LLXML to XLIFF for file ${FILE}"
+                       if [ ! -f $XLF_FILE ]; then
+                               xsltproc --stringparam lang en \
+                                       --stringparam extension ${EXTENSION} \
+                                       --stringparam date "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" ${XSL_LLXML2TEMPLATE $FILE} | \
+                                       xmlstarlet ed --insert "/xliff/file/body/trans-unit" --type attr -n xml:space -v preserve > ${XLF_FILE}
+                       fi
+               fi
+       fi
+       rm -f ${FILE}
+done
+popd >/dev/null
+
+#
+# Deploy new templates files
+echo "[INFO]    Deploy new templates to ${TEMPLATE_DIRECTORY}"
+
+mkdir -p /tmp/backup
+cd ${TEMPLATE_DIRECTORY}/../../ >/dev/null && \
+       tar cfzp /tmp/backup/${PROJECT_NAME}-$(date +%Y%m%d).tar.gz ${PROJECT_NAME} && \
+               cd - >/dev/null && \
+               rm -Rf ${TEMPLATE_DIRECTORY} && \
+               mkdir -p ${TEMPLATE_DIRECTORY} && \
+               cp -a /tmp/${PROJECT_NAME}/* ${TEMPLATE_DIRECTORY} || \
+               onexit 5
+
+#
+# Check project available language
+check_available_languages ${PROJECT_NAME}
+
+# Find new files automatically
+echo "[INFO]    Update stores"
+${POOTLE_MANAGE} update_stores --project=${PROJECT_NAME} --language=templates >/dev/null 2>&1
+
+echo "[INFO]    Update from templates"
+${POOTLE_MANAGE} update_against_templates --project=${PROJECT_NAME} >/dev/null 2>&1
+
+echo "[INFO]    Refresh statistics"
+${POOTLE_MANAGE} refresh_stats --project=${PROJECT_NAME} >/dev/null 2>&1
+
+echo "[NOTICE]  Register version ${VERSION} for EXT:${EXTENSION}"
+echo -n ${VERSION} > ${PROJECT_DIRECTORY}/.ter-version
+
+#
+# Cleanup
+pushd /tmp >/dev/null
+rm -f ${EXTENSION_T3X}
+if [ -d ${EXTENSION} ]; then
+       rm -rf ${EXTENSION}
+       rm -rf TYPO3.ext.${EXTENSION}
+fi
+popd >/dev/null
+
+IFS=$SAVEIFS
+
+onexit
index 47ff14f..fb26043 100755 (executable)
@@ -240,14 +240,19 @@ for PROJECT in ${PROJECTS}; do
        echo "[INFO]    Refreshing Pootle for project ${PROJECT}"
 
        # Find new files automatically
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_stores --project=${PROJECT} --language=templates
+       echo "[INFO]    Update stores"
+       ${POOTLE_MANAGE} update_stores --project=${PROJECT} --language=templates >/dev/null 2>&1
 
        # Add new languages if needed
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_translation_projects --project=${PROJECT}
+       echo "[INFO]    Add new languages"
+       ${POOTLE_MANAGE} update_translation_projects --project=${PROJECT} >/dev/null 2>&1
 
        # Update translations with labels from templates (source language)
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_against_templates --project=${PROJECT}
-       ${POOTLE_SCRIPT_HOME}/bin/pootle-manage refresh_stats --project=${PROJECT}
+       echo "[INFO]    Update from templates"
+       ${POOTLE_MANAGE} update_against_templates --project=${PROJECT} >/dev/null 2>&1
+
+       echo "[INFO]    Refresh statistics"
+       ${POOTLE_MANAGE} refresh_stats --project=${PROJECT} >/dev/null 2>&1
 done
 
 popd >/dev/null
index 2f59db8..9b418b2 100644 (file)
@@ -11,6 +11,9 @@
 - name: Install xsltproc
   apt: pkg=xsltproc state=installed
 
+- name: Install command-line interpreter for php5
+  apt: pkg=php5-cli state=installed
+
 - name: Install Pootle user profile
   copy: src=bash_profile.txt dest=/home/pootle/.bash_profile owner=pootle group=pootle
 
index cb6064a..ed9b0ff 100644 (file)
@@ -2,6 +2,9 @@ POOTLE_BIN={{ pootle_virtualenv }}/bin/pootle
 POOTLE_CONF=/etc/pootle/pootle.conf
 POOTLE_SCRIPT_HOME=/home/pootle/scripts
 
+POOTLE_MANAGE=${POOTLE_SCRIPT_HOME}/bin/pootle-manage
+T3XUTILS=${POOTLE_SCRIPT_HOME}/bin/t3xutils.phar
+
 #POOTLE_PID=$POOTLE_SCRIPT_HOME/tmp/fcgi-t3-pootle.pid
 #POOTLE_FCGI_SOCK=/tmp/fcgi-t3-pootle.sock
 POOTLE_LOG=/var/log/pootle/pootle.log
@@ -16,6 +19,7 @@ LOCAL_GIT_CLONE_DIRECTORY="{{ pootle_git_directory }}"
 
 XSL_LLXML2XLIFF_SOURCE=${POOTLE_SCRIPT_HOME}/etc/xsl/xliff2llxml-source.xsl
 XSL_LLXML2XLIFF_TARGET=${POOTLE_SCRIPT_HOME}/etc/xsl/xliff2llxml-target.xsl
+XSL_LLXML2TEMPLATE=${POOTLE_SCRIPT_HOME}/etc/xsl/llxml2template.xsl
 
 _LAST_BUILD_FILE="${POOTLE_SCRIPT_HOME}/.build-language-pack-lastbuild"