[TASK] Script to update a 3rd party extension

Change-Id: I10ee82c4d2dabde90e488fbb1cbc35b6d45e8c8a
Reviewed-on: http://review.typo3.org/41500

Reviewed-by: Xavier Perseguers <xavier@typo3.org>
Tested-by: Xavier Perseguers <xavier@typo3.org>

LEGACY/bin/pootle-upgrade-extension

-#!/usr/bin/env bash
-
-set -o errtrace
-set -o nounset
-
-#  Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR
-trap onexit 1 2 3 15
-
-#--- onexit() -----------------------------------------------------
-#  @param $1 integer  (optional) Exit status.  If not set, use `$?'
-function onexit() {
-	local exit_status=${1:-$?}
-	if [ $exit_status -gt 0 ]; then
-		echo Exiting $0 with $exit_status
-	fi
-	exit $exit_status
-}
-
-T3XUTILS=/var/www/vhosts/pootle.typo3.org/home/scripts/bin/t3xutils.phar
-POOTLE_MANAGE=/var/www/vhosts/pootle.typo3.org/home/scripts/bin/pootle-manage
-
-# --- check_available_languages() ---------------------------------
-# @param $1 string               Pootle Project Name
-function check_available_languages() {
-	local NEED_UPDATE=0
-	local PROJECT_NAME=$1
-        for LANGUAGE_KEY in $(ls /var/www/vhosts/pootle.typo3.org/home/templates); do
-		local LANGUAGE_DIRECTORY="/var/www/vhosts/pootle.typo3.org/pootle/po/$PROJECT_NAME/$LANGUAGE_KEY"
-                if [ "$PROJECT_NAME" != "" ] && [ -d /var/www/vhosts/pootle.typo3.org/pootle/po/$PROJECT_NAME ] && [ ! -d $LANGUAGE_DIRECTORY ]; then
-                        mkdir -p $LANGUAGE_DIRECTORY
-                        echo "[NOTICE] Add project language \"$LANGUAGE_KEY\""
-			NEED_UPDATE=1
-                fi
-        done
-	if [ $NEED_UPDATE == 1 ]; then
-		echo "[NOTICE] Update translation project"
-		$POOTLE_MANAGE update_translation_projects --project=$1 1>/dev/null 2>/dev/null
-	fi
-}
-
-SAVEIFS=$IFS
-
-XSL_LLXML2TEMPLATE="/var/www/vhosts/pootle.typo3.org/home/scripts/etc/xsl/llxml2template.xsl"
-TEMP_DIRECTORY="/var/www/vhosts/pootle.typo3.org/home/tmp"
-
-EXTENSION=$1
-VERSION=$2
-
-PROJECT_NAME="TYPO3.TYPO3.ext.$EXTENSION"
-PROJECT_DIRECTORY="/var/www/vhosts/pootle.typo3.org/pootle/po/$PROJECT_NAME"
-#
-# Check if the project exist
-if [ ! -d $PROJECT_DIRECTORY ]; then
-	echo
-	echo "[ERROR] Project $PROJECT_NAME doesn't exist in Pootle, please create it before using this script"
-	echo "[ERROR] Please create directory: $PROJECT_DIRECTORY"
-	onexit 1
-fi
-
-#
-# Check if the templates directory exist
-TEMPLATE_DIRECTORY="$PROJECT_DIRECTORY/templates"
-if [ ! -d $TEMPLATE_DIRECTORY ]; then
-        echo
-        echo "[INFO] Templates directory for project $PROJECT_NAME doesn't exist in Pootle"
-	echo "[NOTICE] Create the templates directory"
-	mkdir -p $TEMPLATE_DIRECTORY
-fi
-
-cd $TEMP_DIRECTORY >/dev/null
-
-echo "[INFO] Downloading $EXTENSION $VERSION from TER"
-
-EXTENSION_T3X=$(echo "${EXTENSION}_${VERSION}.t3x" | tr '[:upper:]' '[:lower:]')
-
-#
-# Download t3x
-rm -f $EXTENSION_T3X
-wget -q http://typo3.org/fileadmin/ter/${EXTENSION_T3X:0:1}/${EXTENSION_T3X:1:1}/$EXTENSION_T3X
-
-#
-# Extract t3x
-#extract-t3x $EXTENSION_T3X $EXTENSION > /dev/null
-$T3XUTILS extract $EXTENSION_T3X $EXTENSION >/dev/null
-
-IFS=$(echo -en "\n\b")
-
-#
-# Create directory structure for templates
-for FILE in $(find $EXTENSION/ -name \*.xml); do
-	grep "T3locallang" "$FILE" 1> /dev/null
-	if [ $? == 0 ]; then
-		mkdir -p $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$(dirname $FILE);
-		cp $FILE $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$FILE;
-	fi
-done
-for FILE in $(find $EXTENSION/ -name \*.xlf); do
-	# Count number of dots in filename to skip e.g., de.locallang.xlf
-	DOTS=$(echo -n $(basename $FILE) | sed 's/[^.]//g' | wc -c)
-	if [ $DOTS -eq 1 ]; then
-		mkdir -p $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$(dirname $FILE);
-		cp $FILE $TEMP_DIRECTORY/TYPO3.TYPO3.ext.$FILE;
-	fi
-done
-
-cd $TEMP_DIRECTORY
-for FILE in $(find TYPO3.TYPO3.ext.$EXTENSION -name \*.xml); do
-	grep "T3locallang" "$FILE" 1> /dev/null
-	if [ $? == 0 ]; then
-		XLF_FILE=$(echo $FILE | sed s/.xml$/.xlf/)
-		if [ ! -f $XLF_FILE ]; then
-			echo "[INFO] Convert LLXML to XLIFF for file $FILE"
-			if [ ! -f $XLF_FILE ]; then
-				xsltproc --stringparam lang en \
-					--stringparam extension $EXTENSION \
-					--stringparam date "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" $XSL_LLXML2TEMPLATE $FILE | \
-					xmlstarlet ed --insert "/xliff/file/body/trans-unit" --type attr -n xml:space -v preserve > $XLF_FILE
-			fi
-		fi
-	fi
-	rm -f $FILE
-done
-
-#
-# Deploy new templates files
-echo "[INFO] Deploy new templates to $TEMPLATE_DIRECTORY"
-cd $TEMPLATE_DIRECTORY/../../ && \
-	tar cfzp $TEMP_DIRECTORY/backup/$PROJECT_NAME-$(date +%Y%m%d).tar.gz $PROJECT_NAME && \
-	cd - && \
-	rm -Rf $TEMPLATE_DIRECTORY && \
-	mkdir -p $TEMPLATE_DIRECTORY && \
-	cp -a $TEMP_DIRECTORY/$PROJECT_NAME/* $TEMPLATE_DIRECTORY || \
-		onexit 5
-
-#
-# Check project available language
-check_available_languages $PROJECT_NAME
-
-#
-# Update project from templates
-echo "[INFO] Update from templates"
-$POOTLE_MANAGE update_from_templates --project=$PROJECT_NAME 1>/dev/null 2>/dev/null
-
-echo "[INFO] Refresh statistics"
-$POOTLE_MANAGE refresh_stats --project=$PROJECT_NAME 1>/dev/null 2>/dev/null
-
-echo "[NOTICE] Register version $VERSION for EXT:$EXTENSION"
-echo -n $VERSION > $PROJECT_DIRECTORY/.ter-version
-
-#
-# Cleanup
-cd $TEMP_DIRECTORY
-rm -f $EXTENSION_T3X
-if [ -d $EXTENSION ]; then
-	rm -rf $EXTENSION
-	rm -rf TYPO3.TYPO3.ext.$EXTENSION
-fi
-
-IFS=$SAVEIFS
-
-onexit

typo3/files/scripts/bin/build-language-pack

@@ -18,7 +18,7 @@ cd $(dirname $0)
 #if [ -z "$FORCE_EXTENSION" ]; then
 #	# Sync whole stores, --project seems not totally reliable
 #	echo "Synchronizing all stores ... "
-#	${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores 2>&1
+#	${POOTLE_MANAGE} sync_stores 2>&1
 #fi
 
 for EXTNAME in $EXTENSIONS; do
@@ -27,11 +27,11 @@ for EXTNAME in $EXTENSIONS; do
 	echo "Packaging $EXTNAME"
 	if [ ! -z "$FORCE_EXTENSION" ]; then
 		echo -n "   synchronizing store ... "
-		${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
+		${POOTLE_MANAGE} sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
 		echo "done."
 	fi
 	echo -n "   updating statistics ... "
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
+	${POOTLE_MANAGE} refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
 	echo "done."
 	pushd ${POOTLE_PO}/$EXT_PREFIX.$EXTNAME >/dev/null
 	rm -rf ${TER_L10N_PATH}/_$EXTNAME
@@ -79,7 +79,7 @@ for EXTNAME in $EXTENSIONS; do
 		esac
 
 		echo -n "   processing $LANG ... "
-		#${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
+		#${POOTLE_MANAGE} sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
 
 		# Compute current state
 		STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time

typo3/files/scripts/bin/build-sysext-language-pack

@@ -18,7 +18,7 @@ cd $(dirname $0)
 #if [ -z "$FORCE_EXTENSION" ]; then
 #	# Sync whole stores, --project seems not totally reliable
 #	echo "Synchronizing all stores ... "
-#	${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores 2>&1
+#	${POOTLE_MANAGE} sync_stores 2>&1
 #fi
 
 for EXTNAME in $EXTENSIONS; do
@@ -27,11 +27,11 @@ for EXTNAME in $EXTENSIONS; do
 	echo "Packaging $EXTNAME"
 	if [ ! -z "$FORCE_EXTENSION" ]; then
 		echo -n "   synchronizing store ... "
-		${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
+		${POOTLE_MANAGE} sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
 		echo "done."
 	fi
 	echo -n "   updating statistics ... "
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
+	${POOTLE_MANAGE} refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
 	echo "done."
 	pushd ${POOTLE_PO}/$EXT_PREFIX.$EXTNAME >/dev/null
 
@@ -92,7 +92,7 @@ for EXTNAME in $EXTENSIONS; do
 			esac
 
 			echo -n "   processing $LANG ... "
-			#${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
+			#${POOTLE_MANAGE} sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
 
 			# Compute current state
 			STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time

typo3/files/scripts/bin/migration/1-migrate-typo3-xliff

@@ -11,7 +11,7 @@ for PROJECT in ${PROJECTS}; do
 	pushd ${PROJECT} >/dev/null
 
 	echo "[INFO]    Migrating ${PROJECT}"
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage sync_stores --overwrite --project=${PROJECT}
+	${POOTLE_MANAGE} sync_stores --overwrite --project=${PROJECT}
 
 	LANGUAGES=$(find . -maxdepth 1 -type d | cut -b3-)
 	for LANGUAGE in ${LANGUAGES}; do
@@ -52,7 +52,7 @@ for PROJECT in ${PROJECTS}; do
 	done
 
 	echo "[INFO]    Updating Pootle stores for ${PROJECT}"
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_stores --project=${PROJECT}
+	${POOTLE_MANAGE} update_stores --project=${PROJECT}
 
 	popd >/dev/null
 done

typo3/files/scripts/bin/update-extension

+#!/usr/bin/env bash
+
+. /home/pootle/scripts/etc/pootle.conf
+. /home/pootle/scripts/etc/functions
+
+set -o errtrace
+set -o nounset
+
+#  Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR
+trap onexit 1 2 3 15
+
+#--- onexit() -----------------------------------------------------
+#  @param $1 integer  (optional) Exit status.  If not set, use `$?'
+function onexit() {
+	local exit_status=${1:-$?}
+	if [ $exit_status -gt 0 ]; then
+		echo Exiting $0 with $exit_status
+	fi
+	exit $exit_status
+}
+
+if [ $# -lt 2 ]; then
+	echo "Usage: $0 <extension-key> <version>" >&2
+	exit 1
+fi
+
+SAVEIFS=$IFS
+
+EXTENSION=$(echo $1 | tr '[:upper:]' '[:lower:]')
+VERSION=$2
+PROJECT_NAME="TYPO3.ext.${EXTENSION}"
+PROJECT_DIRECTORY="${POOTLE_PO}/${PROJECT_NAME}"
+
+#
+# Check if the project exist
+if [ ! -d ${PROJECT_DIRECTORY} ]; then
+	echo
+	echo "[ERROR]   Project ${PROJECT_NAME} doesn't exist in Pootle, please create it before using this script"
+	echo "[ERROR]   Please create directory: ${PROJECT_DIRECTORY}"
+	onexit 1
+fi
+
+#
+# Check if the templates directory exist
+TEMPLATE_DIRECTORY="${PROJECT_DIRECTORY}/templates"
+if [ ! -d $TEMPLATE_DIRECTORY ]; then
+        echo
+        echo "[INFO]    Templates directory for project ${PROJECT_NAME} doesn't exist in Pootle"
+	echo "[NOTICE]  Create the templates directory"
+	mkdir -p $TEMPLATE_DIRECTORY
+fi
+
+pushd /tmp >/dev/null
+
+echo "[INFO]    Downloading ${EXTENSION} ${VERSION} from TER"
+
+#
+# Download t3x
+EXTENSION_T3X=${EXTENSION}_${VERSION}.t3x
+rm -f ${EXTENSION_T3X}
+${T3XUTILS} fetch ${EXTENSION} ${VERSION}
+if [ ! -f ${EXTENSION_T3X} ]; then
+	echo "[ERROR]   Could not download extension from TER"
+	onexit 1
+fi
+
+#
+# Extract t3x
+${T3XUTILS} extract ${EXTENSION_T3X} ${EXTENSION} >/dev/null
+
+IFS=$(echo -en "\n\b")
+
+#
+# Create directory structure for templates
+for FILE in $(find ${EXTENSION}/ -name \*.xml); do
+	grep "T3locallang" "${FILE}" >/dev/null
+	if [ $? -eq 0 ]; then
+		mkdir -p /tmp/TYPO3.ext.$(dirname ${FILE})
+		cp ${FILE} /tmp/TYPO3.ext.${FILE}
+	fi
+done
+for FILE in $(find ${EXTENSION}/ -name \*.xlf); do
+	# Count number of dots in filename to skip e.g., de.locallang.xlf
+	DOTS=$(echo -n $(basename $FILE) | sed 's/[^.]//g' | wc -c)
+	if [ ${DOTS} -eq 1 ]; then
+		mkdir -p /tmp/TYPO3.ext.$(dirname ${FILE})
+		cp ${FILE} /tmp/TYPO3.ext.${FILE}
+	fi
+done
+
+for FILE in $(find TYPO3.ext.${EXTENSION} -name \*.xml); do
+	grep "T3locallang" "${FILE}" >/dev/null
+	if [ $? -eq 0 ]; then
+		XLF_FILE=$(echo ${FILE} | sed s/.xml$/.xlf/)
+		if [ ! -f ${XLF_FILE} ]; then
+			echo "[INFO]    Convert LLXML to XLIFF for file ${FILE}"
+			if [ ! -f $XLF_FILE ]; then
+				xsltproc --stringparam lang en \
+					--stringparam extension ${EXTENSION} \
+					--stringparam date "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" ${XSL_LLXML2TEMPLATE $FILE} | \
+					xmlstarlet ed --insert "/xliff/file/body/trans-unit" --type attr -n xml:space -v preserve > ${XLF_FILE}
+			fi
+		fi
+	fi
+	rm -f ${FILE}
+done
+popd >/dev/null
+
+#
+# Deploy new templates files
+echo "[INFO]    Deploy new templates to ${TEMPLATE_DIRECTORY}"
+
+mkdir -p /tmp/backup
+cd ${TEMPLATE_DIRECTORY}/../../ >/dev/null && \
+	tar cfzp /tmp/backup/${PROJECT_NAME}-$(date +%Y%m%d).tar.gz ${PROJECT_NAME} && \
+		cd - >/dev/null && \
+		rm -Rf ${TEMPLATE_DIRECTORY} && \
+		mkdir -p ${TEMPLATE_DIRECTORY} && \
+		cp -a /tmp/${PROJECT_NAME}/* ${TEMPLATE_DIRECTORY} || \
+		onexit 5
+
+#
+# Check project available language
+check_available_languages ${PROJECT_NAME}
+
+# Find new files automatically
+echo "[INFO]    Update stores"
+${POOTLE_MANAGE} update_stores --project=${PROJECT_NAME} --language=templates >/dev/null 2>&1
+
+echo "[INFO]    Update from templates"
+${POOTLE_MANAGE} update_against_templates --project=${PROJECT_NAME} >/dev/null 2>&1
+
+echo "[INFO]    Refresh statistics"
+${POOTLE_MANAGE} refresh_stats --project=${PROJECT_NAME} >/dev/null 2>&1
+
+echo "[NOTICE]  Register version ${VERSION} for EXT:${EXTENSION}"
+echo -n ${VERSION} > ${PROJECT_DIRECTORY}/.ter-version
+
+#
+# Cleanup
+pushd /tmp >/dev/null
+rm -f ${EXTENSION_T3X}
+if [ -d ${EXTENSION} ]; then
+	rm -rf ${EXTENSION}
+	rm -rf TYPO3.ext.${EXTENSION}
+fi
+popd >/dev/null
+
+IFS=$SAVEIFS
+
+onexit

typo3/files/scripts/bin/update-typo3-core-template

@@ -240,14 +240,19 @@ for PROJECT in ${PROJECTS}; do
 	echo "[INFO]    Refreshing Pootle for project ${PROJECT}"
 
 	# Find new files automatically
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_stores --project=${PROJECT} --language=templates
+	echo "[INFO]    Update stores"
+	${POOTLE_MANAGE} update_stores --project=${PROJECT} --language=templates >/dev/null 2>&1
 
 	# Add new languages if needed
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_translation_projects --project=${PROJECT}
+	echo "[INFO]    Add new languages"
+	${POOTLE_MANAGE} update_translation_projects --project=${PROJECT} >/dev/null 2>&1
 
 	# Update translations with labels from templates (source language)
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage update_against_templates --project=${PROJECT}
-	${POOTLE_SCRIPT_HOME}/bin/pootle-manage refresh_stats --project=${PROJECT}
+	echo "[INFO]    Update from templates"
+	${POOTLE_MANAGE} update_against_templates --project=${PROJECT} >/dev/null 2>&1
+
+	echo "[INFO]    Refresh statistics"
+	${POOTLE_MANAGE} refresh_stats --project=${PROJECT} >/dev/null 2>&1
 done
 
 popd >/dev/null

typo3/tasks/main.yml

@@ -11,6 +11,9 @@
 - name: Install xsltproc
   apt: pkg=xsltproc state=installed
 
+- name: Install command-line interpreter for php5
+  apt: pkg=php5-cli state=installed
+
 - name: Install Pootle user profile
   copy: src=bash_profile.txt dest=/home/pootle/.bash_profile owner=pootle group=pootle
 

typo3/templates/scripts/etc/pootle.conf.j2

@@ -2,6 +2,9 @@ POOTLE_BIN={{ pootle_virtualenv }}/bin/pootle
 POOTLE_CONF=/etc/pootle/pootle.conf
 POOTLE_SCRIPT_HOME=/home/pootle/scripts
 
+POOTLE_MANAGE=${POOTLE_SCRIPT_HOME}/bin/pootle-manage
+T3XUTILS=${POOTLE_SCRIPT_HOME}/bin/t3xutils.phar
+
 #POOTLE_PID=$POOTLE_SCRIPT_HOME/tmp/fcgi-t3-pootle.pid
 #POOTLE_FCGI_SOCK=/tmp/fcgi-t3-pootle.sock
 POOTLE_LOG=/var/log/pootle/pootle.log
@@ -16,6 +19,7 @@ LOCAL_GIT_CLONE_DIRECTORY="{{ pootle_git_directory }}"
 
 XSL_LLXML2XLIFF_SOURCE=${POOTLE_SCRIPT_HOME}/etc/xsl/xliff2llxml-source.xsl
 XSL_LLXML2XLIFF_TARGET=${POOTLE_SCRIPT_HOME}/etc/xsl/xliff2llxml-target.xsl
+XSL_LLXML2TEMPLATE=${POOTLE_SCRIPT_HOME}/etc/xsl/llxml2template.xsl
 
 _LAST_BUILD_FILE="${POOTLE_SCRIPT_HOME}/.build-language-pack-lastbuild"