[TASK] Script to package TYPO3 system extensions

Change-Id: I7f97e213dc689fd083fb7313024a9548f3446f54
Reviewed-on: http://review.typo3.org/41380

Reviewed-by: Xavier Perseguers <xavier@typo3.org>
Tested-by: Xavier Perseguers <xavier@typo3.org>

typo3/files/bash_profile.txt

@@ -4,7 +4,7 @@ export PATH
 echo
 echo "Welcome TYPO3 Pootle Manager"
 echo "============================"
-echo 
+echo
 echo "Administration command:"
-find ~/scripts/bin -maxdepth 1 -type f -perm +111 -exec basename {} \; | awk '{ print "\t"$1 }'
+find ~/scripts/bin -maxdepth 1 -type f -perm +111 -exec basename {} \; | awk '{ print "\t"$1 }' | sort
 echo
\ No newline at end of file

typo3/files/scripts/bin/build-language-pack

@@ -22,6 +22,8 @@ cd $(dirname $0)
 #fi
 
 for EXTNAME in $EXTENSIONS; do
+	validProjectName ${EXT_PREFIX}.${EXTNAME} || exit 1
+
 	echo "Packaging $EXTNAME"
 	if [ ! -z "$FORCE_EXTENSION" ]; then
 		echo -n "   synchronizing store ... "
@@ -91,7 +93,7 @@ for EXTNAME in $EXTENSIONS; do
 			mkdir -p ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME
 			cp -r $ORIG_LANG/* ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME
 			pushd ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME >/dev/null
-			for XLF in $(find . -type f); do
+			for XLF in $(find . -type f | cut -b3-); do
 				DIR=$(dirname $XLF)
 				FILE=$(basename $XLF)
 				mv $XLF $DIR/$LANG.$FILE

typo3/files/scripts/bin/build-sysext-language-pack

+#!/usr/bin/env bash
+
+. /home/pootle/scripts/etc/pootle.conf
+. /home/pootle/scripts/etc/functions
+
+# -- no need to modify anything below
+
+FORCE_EXTENSION=$1
+EXT_PREFIX=TYPO3.TYPO3.core
+EXT_PREFIX_LENGTH=$(($(echo -n ${EXT_PREFIX} | wc -c) + 2))
+if [ ! -z "${FORCE_EXTENSION}" ]; then
+	EXTENSIONS=${FORCE_EXTENSION}
+else
+	EXTENSIONS=$(ls ${POOTLE_PO} | grep ${EXT_PREFIX} | cut -b${EXT_PREFIX_LENGTH}-)
+fi
+
+cd $(dirname $0)
+#if [ -z "$FORCE_EXTENSION" ]; then
+#	# Sync whole stores, --project seems not totally reliable
+#	echo "Synchronizing all stores ... "
+#	./pootle-manage sync_stores 2>&1
+#fi
+
+for EXTNAME in $EXTENSIONS; do
+	validProjectName ${EXT_PREFIX}.${EXTNAME} || exit 1
+
+	echo "Packaging $EXTNAME"
+	if [ ! -z "$FORCE_EXTENSION" ]; then
+		echo -n "   synchronizing store ... "
+		./pootle-manage sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
+		echo "done."
+	fi
+	echo -n "   updating statistics ... "
+	./pootle-manage refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
+	echo "done."
+	pushd ${POOTLE_PO}/$EXT_PREFIX.$EXTNAME >/dev/null
+
+	_LANGUAGE_PACK_PATH=${TER_L10N_PATH}/${EXTNAME:0:1}/${EXTNAME:1:1}/${EXTNAME}-l10n
+	mkdir -p $_LANGUAGE_PACK_PATH
+	mkdir -p $STATE_DIR/$EXT_PREFIX.$EXTNAME
+
+	for VERSION in $(find .typo3 -name \*.filemapping | sed -r 's/^.*TYPO3(v.)\.filemapping/\1/'); do
+		echo "   ----------------------"
+		echo "   Processsing TYPO3 ${VERSION}.x"
+		echo "   ----------------------"
+
+		rm -rf ${TER_L10N_PATH}/_$EXTNAME
+		LANGUAGES="$(ls | grep -v templates)"
+
+		SUFFIX=""
+		if [ "${VERSION}" = 'v6' ]; then
+			LANGUAGES="${LANGUAGES} ba br ch cz dk si se gr hk kr ua jp qc vn ge ga"
+		else
+			SUFFIX=".${VERSION}"
+		fi
+
+		for LANG in $LANGUAGES; do
+			ORIG_LANG=$LANG
+			case "$LANG" in
+				'ba')
+					ORIG_LANG=bs ;;
+				'br')
+					ORIG_LANG=pt_BR ;;
+				'ch')
+					ORIG_LANG=zh_CN ;;
+				'cz')
+					ORIG_LANG=cs ;;
+				'dk')
+					ORIG_LANG=da ;;
+				'si')
+					ORIG_LANG=sl ;;
+				'se')
+					ORIG_LANG=sv ;;
+				'gr')
+					ORIG_LANG=el ;;
+				'hk')
+					ORIG_LANG=zh_HK ;;
+				'kr')
+					ORIG_LANG=ko ;;
+				'ua')
+					ORIG_LANG=uk ;;
+				'jp')
+					ORIG_LANG=ja ;;
+				'qc')
+					ORIG_LANG=fr_CA ;;
+				'vn')
+					ORIG_LANG=vi ;;
+				'ge')
+					ORIG_LANG=ka ;;
+				'ga')
+					ORIG_LANG=gl ;;
+			esac
+
+			echo -n "   processing $LANG ... "
+			#./pootle-manage sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
+
+			# Compute current state
+			STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time
+			if [[ -f $STATUS_FILE && -z "$FORCE_EXTENSION" ]]; then
+				LAST_GENERATION=$(cat $STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time)
+			else
+				LAST_GENERATION=0
+			fi
+			LAST_EDIT=$(lastedit "$EXT_PREFIX.$EXTNAME" $ORIG_LANG)
+			if [[ $LAST_EDIT -gt 0 && $LAST_EDIT -ne $LAST_GENERATION ]]; then
+				mkdir -p ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME
+				cp -r $ORIG_LANG/* ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME
+				pushd ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME >/dev/null
+				for XLF in $(find . -type f | cut -b3-); do
+					MAPPING=${POOTLE_PO}/${EXT_PREFIX}.${EXTNAME}/.typo3/TYPO3${VERSION}.filemapping
+					if [ ! -f "${MAPPING}" ]; then
+						continue
+					fi
+					TARGET_FILE=$(grep $XLF ${MAPPING} | cut -d' ' -f2)
+					if [ -z "${TARGET_FILE}" ]; then
+						# File does not exist for current TYPO3 version
+						continue
+					fi
+
+					T3UID=$(echo $XLF | sed -r 's/^locallang\.(.*)\.xlf$/\1/')
+					KEYS=${POOTLE_PO}/${EXT_PREFIX}.${EXTNAME}/.typo3/TYPO3${VERSION}.${T3UID}.keys
+					if [ ! -r "$KEYS" ]; then
+						# Strange!
+						continue
+					fi
+
+					DIR=$(dirname ${TARGET_FILE})
+					FILE=$(basename ${TARGET_FILE})
+					TRANSLATED_TARGET_FILE=${DIR}/${LANG}.$FILE
+
+					# Create a blank XLF
+					mkdir -p ${DIR}
+
+					cat <<EOT > ${TRANSLATED_TARGET_FILE}
+<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
+<xliff version="1.0">
+	<file source-language="en" target-language="${LANG}" datatype="plaintext" original="messages" date="$(date +"%FT%H:%M:%SZ")" product-name="${EXTNAME}">
+		<header/>
+		<body>
+EOT
+
+					# Important to use this construct in order to handle keys
+					# with blank spaces (EXT:rtehtmlarea)
+					cat $KEYS | while read KEY ; do
+						echo -en "\t\t\t" >> ${TRANSLATED_TARGET_FILE}
+						xmlstarlet sel -t -c "//trans-unit[@id='${KEY}']" ${XLF} | sed -e 's/ xmlns:t3="[^"]*"//g' >> ${TRANSLATED_TARGET_FILE}
+						echo >> ${TRANSLATED_TARGET_FILE}
+					done
+
+					cat <<EOT >> ${TRANSLATED_TARGET_FILE}
+		</body>
+	</file>
+</xliff>
+EOT
+
+					# Reformat XLF with proper indent
+					xmlstarlet fo -t ${TRANSLATED_TARGET_FILE} > ${TRANSLATED_TARGET_FILE}.tmp
+					mv ${TRANSLATED_TARGET_FILE}.tmp ${TRANSLATED_TARGET_FILE}
+
+					if [ "${VERSION}" = 'v6' ]; then
+						# Convert XLIFF to LLXML
+						_SOURCE_XLIFF_FILE=${POOTLE_PO}/${EXT_PREFIX}.${EXTNAME}/templates/${DIR}/${FILE}
+						LLXML="$(echo ${TRANSLATED_TARGET_FILE} | sed s/\.xlf$/\.xml/)"
+						llxml2xliff $EXTNAME \
+							$LANG \
+							$_SOURCE_XLIFF_FILE \
+							${TRANSLATED_TARGET_FILE} > $LLXML
+					fi
+				done
+				find . -name \*.xlf -exec chmod 644 {} \;
+				find . -name \*.xml -exec chmod 644 {} \;
+				rm -f ${_LANGUAGE_PACK_PATH}/${EXTNAME}-l10n-${LANG}${SUFFIX}.zip
+				# ../ to include $EXTNAME directory
+				cd ../
+				zip -9 ${_LANGUAGE_PACK_PATH}/${EXTNAME}-l10n-${LANG}${SUFFIX}.zip -r . >/dev/null
+
+				popd >/dev/null
+
+				# Save current state
+				echo $LAST_EDIT > $STATUS_FILE
+				echo "done."
+			else
+				echo "skipped."
+			fi
+		done
+
+		# Create language pack index
+		pushd $_LANGUAGE_PACK_PATH >/dev/null
+
+		_LANGUAGE_PACK_NAME=${EXTNAME}-l10n
+		_LANGUAGE_INDEX_FILE=${_LANGUAGE_PACK_NAME}${SUFFIX}.xml
+		rm -f $_LANGUAGE_INDEX_FILE
+		touch $_LANGUAGE_INDEX_FILE
+
+		echo -e "<?xml version=\"1.0\" standalone=\"yes\" ?>"                   >> $_LANGUAGE_INDEX_FILE
+		echo -e "<TERlanguagePackIndex>"                                        >> $_LANGUAGE_INDEX_FILE
+		echo -e "\t<meta>"                                                      >> $_LANGUAGE_INDEX_FILE
+		echo -e "\t\t<timestamp>$(date +"%s")</timestamp>"                      >> $_LANGUAGE_INDEX_FILE
+		echo -e "\t\t<date>$(date +"%F %T")</date>"                             >> $_LANGUAGE_INDEX_FILE
+		echo -e "\t</meta>"                                                     >> $_LANGUAGE_INDEX_FILE
+		echo -e "\t<languagePackIndex>"                                         >> $_LANGUAGE_INDEX_FILE
+
+		for p in $(find . -name \*${SUFFIX}.zip | sort);
+		do
+			_LANGUAGE_KEY=$(echo $p | sed -r "s/.*${_LANGUAGE_PACK_NAME}-(.*)${SUFFIX}\.zip\$/\1/")
+			if [ -n "${SUFFIX}" ] || [ -z "$(echo ${_LANGUAGE_KEY} | grep '\.')" ]; then
+				echo -e "\t\t<languagepack language=\"$_LANGUAGE_KEY\">"        >> $_LANGUAGE_INDEX_FILE
+				echo -e "\t\t\t<md5>$(md5sum $p | cut -d" " -f1)</md5>"         >> $_LANGUAGE_INDEX_FILE
+				echo -e "\t\t</languagepack>"                                   >> $_LANGUAGE_INDEX_FILE
+			fi
+		done
+
+		echo -e "\t</languagePackIndex>"                                        >> $_LANGUAGE_INDEX_FILE
+		echo -e "</TERlanguagePackIndex>"                                       >> $_LANGUAGE_INDEX_FILE
+
+		popd >/dev/null
+
+		rm -rf ${TER_L10N_PATH}/_$EXTNAME
+	done
+
+	popd >/dev/null
+done

typo3/files/scripts/bin/migration/migrate-typo3-xliff

 #!/usr/bin/env bash
 
-. ~/scripts/etc/pootle.conf
+. /home/pootle/scripts/etc/pootle.conf
 
 pushd ${POOTLE_PO} >/dev/null