Skip to content

GitLab

Commit 27d587fa authored by Xavier Perseguers's avatar Xavier Perseguers
Browse files

[TASK] Script to package 3rd party TYPO3 extensions 

Change-Id: I23e194af7c5281baa494261d1b1925101680be7a
Reviewed-on: http://review.typo3.org/41368

Reviewed-by: Xavier Perseguers's avatarXavier Perseguers <xavier@typo3.org>
Tested-by: Xavier Perseguers's avatarXavier Perseguers <xavier@typo3.org>
parent cdc8aef0
Hide whitespace changes
Inline Side-by-side
LEGACY/bin/pootle-batch-check-project
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -e
......
LEGACY/bin/pootle-batch-import-projects
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
. /var/www/vhosts/pootle.typo3.org/home/scripts/etc/pootle.conf
. /var/www/vhosts/pootle.typo3.org/home/scripts/etc/functions
......
LEGACY/bin/pootle-check-project
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -e
......
LEGACY/bin/pootle-create-project
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -e
......
LEGACY/bin/pootle-notify-hassuggestion
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
POOTLE_CONFIG=/var/www/vhosts/pootle.typo3.org/pootle/localsettings.py
......
LEGACY/bin/pootle-refresh-project
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -e
......
LEGACY/bin/pootle-snapshot
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
SNAPSHOT_DIRECTORY="/var/www/vhosts/pootle.typo3.org/home/archives/snapshot"
......
LEGACY/bin/pootle-upgrade-all-extensions
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -o errtrace
set -o nounset
......
LEGACY/bin/pootle-upgrade-extension
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -o errtrace
set -o nounset
......
LEGACY/bin/pootle-upgrade-extension-git
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -o errtrace
set -o nounset
......
nginx/templates/virtualhost.conf.j2
View file @ 27d587fa
......@@ -22,6 +22,11 @@ server {
root {{ pootle_virtualenv }}/lib/python2.7/site-packages/pootle/;
}
location ^~ /l10n_ter/ {
root /var/www/{{ domain }}/public/;
autoindex on;
}
location / {
fastcgi_pass 127.0.0.1:8080;
fastcgi_param QUERY_STRING $query_string;
......
LEGACY/bin/build-language-pack → typo3/files/scripts/bin/build-language-pack
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
BASE_DIR=/var/www/vhosts/pootle.typo3.org/pootle/po
EXPORT_DIR=/var/www/vhosts/pootle.typo3.org/l10n_ter
STATE_DIR=/var/www/vhosts/pootle.typo3.org/home/scripts/.build-language-pack-state
POOTLE_CONFIG=/var/www/vhosts/pootle.typo3.org/pootle/localsettings.py
. /home/pootle/scripts/etc/pootle.conf
. /home/pootle/scripts/etc/functions
# -- no need to modify anything below
FORCE_EXTENSION=$1
EXT_PREFIX=TYPO3.TYPO3
EXTENSIONS=$(ls $BASE_DIR | grep $EXT_PREFIX | cut -b13-)
DATABASE_NAME=$(grep ^DATABASE_NAME $POOTLE_CONFIG | sed -r "s/.* = '([^']+)'.*/\1/")
DATABASE_USER=$(grep ^DATABASE_USER $POOTLE_CONFIG | sed -r "s/.* = '([^']+)'.*/\1/")
DATABASE_PASSWORD=$(grep ^DATABASE_PASSWORD $POOTLE_CONFIG | sed -r "s/.* = '([^']+)'.*/\1/")
DATABASE_HOST=$(grep ^DATABASE_HOST $POOTLE_CONFIG | sed -r "s/.* = '([^']+)'.*/\1/")
if [ ! -z "$FORCE_EXTENSION" ]; then
EXTENSIONS=$FORCE_EXTENSION
EXT_PREFIX=TYPO3.TYPO3.ext
EXT_PREFIX_LENGTH=$(($(echo -n ${EXT_PREFIX} | wc -c) + 2))
if [ ! -z "${FORCE_EXTENSION}" ]; then
EXTENSIONS=${FORCE_EXTENSION}
else
EXTENSIONS=$(ls ${POOTLE_PO} | grep ${EXT_PREFIX} | cut -b${EXT_PREFIX_LENGTH}-)
fi
. /var/www/vhosts/pootle.typo3.org/home/scripts/etc/functions
# Usage: mysqlpipe "sql-query"
function mysqlpipe() {
echo $@ | mysql -u $DATABASE_USER -p"$DATABASE_PASSWORD" -h $DATABASE_HOST $DATABASE_NAME | sed '1d'
}
# Usage: lastedit "project" "language"
function lastedit() {
local PROJECT="$1"
local LANGUAGE="$2"
# XP/19.06.2013: Using filesystem instead of database
#timestamp=$(mysqlpipe "SELECT p.id, p.code, l.code AS lang, u.mtime, UNIX_TIMESTAMP(u.mtime) AS tstamp
# FROM pootle_app_project AS p
# LEFT JOIN pootle_app_translationproject as tp ON p.id = tp.project_id
# LEFT JOIN pootle_store_store AS s ON tp.id = s.translation_project_id
# LEFT JOIN pootle_store_unit AS u ON s.id = u.store_id
# INNER JOIN pootle_app_language l ON l.id = tp.language_id
# WHERE ( u.state = 50 OR u.state = 200 )
# AND p.code = '$PROJECT'
# AND l.code = '$LANGUAGE'
# GROUP BY p.id, l.code;" \
#| awk '{ print $6 }')
LANGUAGE_DIR=$BASE_DIR/$PROJECT/$LANGUAGE
if [ -d $LANGUAGE_DIR ]; then
timestamp=$(find $LANGUAGE_DIR -type f -printf '%T@ %p\n' \
| grep -v "./.translation_index" \
| sort -n \
| tail -n 1 \
| cut -f1 -d.
)
fi
if [ -z "$timestamp" ]; then
echo 0
else
echo $timestamp
fi
}
cd $(dirname $0)
if [ -z "$FORCE_EXTENSION" ]; then
# Sync whole stores, --project seems not totally reliable
echo "Synchronizing all stores ... "
./pootle-manage sync_stores 2>&1
fi
#if [ -z "$FORCE_EXTENSION" ]; then
# # Sync whole stores, --project seems not totally reliable
# echo "Synchronizing all stores ... "
# ./pootle-manage sync_stores 2>&1
#fi
for EXT in $EXTENSIONS; do
EXTNAME=$(echo $EXT | sed -r 's/^[^.]+\.//')
for EXTNAME in $EXTENSIONS; do
echo "Packaging $EXTNAME"
if [ ! -z "$FORCE_EXTENSION" ]; then
echo -n " synchronizing store ... "
./pootle-manage sync_stores --project=$EXT_PREFIX.$EXT >/dev/null 2>&1
./pootle-manage sync_stores --project=$EXT_PREFIX.$EXTNAME >/dev/null 2>&1
echo "done."
fi
echo -n " updating statistics ... "
./pootle-manage refresh_stats --project="$EXT_PREFIX.$EXT" >/dev/null 2>&1
./pootle-manage refresh_stats --project="$EXT_PREFIX.$EXTNAME" >/dev/null 2>&1
echo "done."
pushd $BASE_DIR/$EXT_PREFIX.$EXT >/dev/null
rm -rf $EXPORT_DIR/_$EXTNAME
pushd ${POOTLE_PO}/$EXT_PREFIX.$EXTNAME >/dev/null
rm -rf ${TER_L10N_PATH}/_$EXTNAME
_LANGUAGE_PACK_PATH=${EXPORT_DIR}/${EXTNAME:0:1}/${EXTNAME:1:1}/${EXTNAME}-l10n
_LANGUAGE_PACK_PATH=${TER_L10N_PATH}/${EXTNAME:0:1}/${EXTNAME:1:1}/${EXTNAME}-l10n
mkdir -p $_LANGUAGE_PACK_PATH
mkdir -p $STATE_DIR/$EXT_PREFIX.$EXT
mkdir -p $STATE_DIR/$EXT_PREFIX.$EXTNAME
LANGUAGES="$(ls | grep -v templates) ba br ch cz dk si se gr hk kr ua jp qc vn ge ga"
for LANG in $LANGUAGES; do
......@@ -125,27 +77,27 @@ for EXT in $EXTENSIONS; do
esac
echo -n " processing $LANG ... "
#/home/pootle/scripts/pootle-manage sync_stores --project="$EXT_PREFIX.$EXT" --language=$ORIG_LANG >/dev/null 2>&1
#./pootle-manage sync_stores --project="$EXT_PREFIX.$EXTNAME" --language=$ORIG_LANG >/dev/null 2>&1
# Compute current state
STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXT/$LANG.time
STATUS_FILE=$STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time
if [[ -f $STATUS_FILE && -z "$FORCE_EXTENSION" ]]; then
LAST_GENERATION=$(cat $STATE_DIR/$EXT_PREFIX.$EXT/$LANG.time)
LAST_GENERATION=$(cat $STATE_DIR/$EXT_PREFIX.$EXTNAME/$LANG.time)
else
LAST_GENERATION=0
fi
LAST_EDIT=$(lastedit "$EXT_PREFIX.$EXT" $ORIG_LANG)
LAST_EDIT=$(lastedit "$EXT_PREFIX.$EXTNAME" $ORIG_LANG)
if [[ $LAST_EDIT -gt 0 && $LAST_EDIT -ne $LAST_GENERATION ]]; then
mkdir -p $EXPORT_DIR/_$EXTNAME/$LANG/$EXTNAME
cp -r $ORIG_LANG/* $EXPORT_DIR/_$EXTNAME/$LANG/$EXTNAME
pushd $EXPORT_DIR/_$EXTNAME/$LANG/$EXTNAME >/dev/null
mkdir -p ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME
cp -r $ORIG_LANG/* ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME
pushd ${TER_L10N_PATH}/_$EXTNAME/$LANG/$EXTNAME >/dev/null
for XLF in $(find . -type f); do
DIR=$(dirname $XLF)
FILE=$(basename $XLF)
mv $XLF $DIR/$LANG.$FILE
# Convert XLIFF to LLXML
_SOURCE_XLIFF_FILE=$BASE_DIR/$EXT_PREFIX.$EXT/templates/$DIR/$FILE
_SOURCE_XLIFF_FILE=${POOTLE_PO}/$EXT_PREFIX.$EXTNAME/templates/$DIR/$FILE
LLXML=$DIR/$LANG.$(echo $FILE | sed s/\.xlf$/\.xml/)
llxml2xliff $EXTNAME \
$LANG \
......@@ -198,6 +150,6 @@ for EXT in $EXTENSIONS; do
popd >/dev/null
rm -rf $EXPORT_DIR/_$EXTNAME
rm -rf ${TER_L10N_PATH}/_$EXTNAME
popd >/dev/null
done
typo3/files/scripts/bin/pootle-manage
View file @ 27d587fa
#!/bin/bash
#!/usr/bin/env bash
set -e
......
typo3/files/scripts/bin/update-typo3-core-template
View file @ 27d587fa
#!/usr/bin/env bash
. ~/scripts/etc/pootle.conf
. ~/scripts/etc/functions
. /home/pootle/scripts/etc/pootle.conf
. /home/pootle/scripts/etc/functions
# Clone of the TYPO3 CMS sources
SOURCES=${LOCAL_GIT_CLONE_DIRECTORY}/TYPO3.CMS
cd $(dirname $0)
# Usage; check_typo3_typo3_core_projects_languages
function check_typo3_typo3_core_projects_languages() {
echo
......@@ -238,14 +240,14 @@ for PROJECT in ${PROJECTS}; do
echo "[INFO] Refreshing Pootle for project ${PROJECT}"
# Find new files automatically
pootle-manage update_stores --project=${PROJECT} --language=templates
./pootle-manage update_stores --project=${PROJECT} --language=templates
# Add new languages if needed
pootle-manage update_translation_projects --project=${PROJECT}
./pootle-manage update_translation_projects --project=${PROJECT}
# Update translations with labels from templates (source language)
pootle-manage update_against_templates --project=${PROJECT}
pootle-manage refresh_stats --project=${PROJECT}
./pootle-manage update_against_templates --project=${PROJECT}
./pootle-manage refresh_stats --project=${PROJECT}
done
popd >/dev/null
......
typo3/files/scripts/etc/functions
View file @ 27d587fa
......@@ -21,7 +21,12 @@ function log_notice() {
# Usage: mysqlpipe "sql-query"
function mysqlpipe() {
echo $@ | mysql pootle | sed '1d'
local DATABASE_NAME=$(grep ^DATABASE_NAME ${POOTLE_CONF} | sed -r "s/.* = '([^']+)'.*/\1/")
local DATABASE_USER=$(grep ^DATABASE_USER ${POOTLE_CONF} | sed -r "s/.* = '([^']+)'.*/\1/")
local DATABASE_PASSWORD=$(grep ^DATABASE_PASSWORD ${POOTLE_CONF} | sed -r "s/.* = '([^']+)'.*/\1/")
local DATABASE_HOST=$(grep ^DATABASE_HOST ${POOTLE_CONF} | sed -r "s/.* = '([^']+)'.*/\1/")
echo $@ | mysql -u ${DATABASE_USER} -p"${DATABASE_PASSWORD}" -h ${DATABASE_HOST} ${DATABASE_NAME} | sed '1d'
}
#
......@@ -34,7 +39,7 @@ function llxml2xliff() {
local SOURCE=$3
local TARGET=$4
if [ "$EXTENSION" = "" ] || [ "$LANG" = "" ] || [ "$SOURCE" = "" ] || [ "$TARGET" = "" ]; then
if [ -z "$EXTENSION" ] || [ -z "$LANG" ] || [ -z "$SOURCE" ] || [ -z "$TARGET" ]; then
echo
msg "Please provide source and target XLIFF. Unable to convert XLIFF to LLXML."
echo
......@@ -64,7 +69,7 @@ function list_typo3_typo3_core_projects() {
# Usage: project_exist project.name
function project_exist() {
echo -n "[INFO] Check if project \"$1\" exists: "
if [ "$1" != "" ] && [ -d ${POOTLE_PO}/$1 ]; then
if [ -n "$1" ] && [ -d ${POOTLE_PO}/$1 ]; then
echo "OK"
if [ ! -d ${POOTLE_PO}/$1/templates ]; then
mkdir -p ${POOTLE_PO}/$1/templates
......@@ -80,7 +85,7 @@ function project_exist() {
# Usage: check_available_languages project.name
function check_available_languages() {
for LANGUAGE_KEY in $(cat ~/templates/languages.txt); do
if [ "$1" != "" ] && [ ! -d ${POOTLE_PO}/$1/$LANGUAGE_KEY ]; then
if [ -n "$1" ] && [ ! -d ${POOTLE_PO}/$1/$LANGUAGE_KEY ]; then
mkdir -p $POOTLE_PO/$1/$LANGUAGE_KEY
echo "[NOTICE] Create project language \"$LANGUAGE_KEY\" for \"$1\""
pootle-manage update_translation_projects --project=$1 --language=$LANGUAGE_KEY
......@@ -95,12 +100,12 @@ function getProjectUpdatedSince() {
local SINCE=$1
local PROJECT_ONLY=$2
if [ "$SINCE" = "" ]; then
if [ -z "$SINCE" ]; then
echo "[ERROR] Empty date, unable to select updated project" >&2
exit 1
fi
if [ "$PROJECT_ONLY" = "" ]; then
if [ -z "$PROJECT_ONLY" ]; then
mysqlpipe "SELECT p.id, p.code, u.mtime, UNIX_TIMESTAMP(u.mtime) AS tstamp
FROM pootle_app_project AS p
LEFT JOIN pootle_app_translationproject as tp ON p.id = tp.project_id
......@@ -126,12 +131,12 @@ function getProjectUpdatedSince() {
# Usage: getLanguageProjectUpdatedSince project-id 2011-07-14
function getLanguageProjectUpdatedSince() {
local PROJECT_ID=$1
if [ "$PROJECT_ID" = "" ]; then
if [ -z "$PROJECT_ID" ]; then
echo "[ERROR] Empty project id, unable to select updated language project" >&2
exit 1
fi
local SINCE=$2
if [ "$SINCE" = "" ]; then
if [ -z "$SINCE" ]; then
echo "[ERROR] Empty date, unable to select updated language project" >&2
exit 1
fi
......@@ -147,6 +152,41 @@ function getLanguageProjectUpdatedSince() {
ORDER BY u.mtime DESC;"
}
# Usage: lastedit "project" "language"
function lastedit() {
local PROJECT="$1"
local LANGUAGE="$2"
# XP/19.06.2013: Using filesystem instead of database
#timestamp=$(mysqlpipe "SELECT p.id, p.code, l.code AS lang, u.mtime, UNIX_TIMESTAMP(u.mtime) AS tstamp
# FROM pootle_app_project AS p
# LEFT JOIN pootle_app_translationproject as tp ON p.id = tp.project_id
# LEFT JOIN pootle_store_store AS s ON tp.id = s.translation_project_id
# LEFT JOIN pootle_store_unit AS u ON s.id = u.store_id
# INNER JOIN pootle_app_language l ON l.id = tp.language_id
# WHERE ( u.state = 50 OR u.state = 200 )
# AND p.code = '$PROJECT'
# AND l.code = '$LANGUAGE'
# GROUP BY p.id, l.code;" \
#| awk '{ print $6 }')
LANGUAGE_DIR=${POOTLE_PO}/${PROJECT}/${LANGUAGE}
if [ -d ${LANGUAGE_DIR} ]; then
timestamp=$(find ${LANGUAGE_DIR} -type f -printf '%T@ %p\n' \
| grep -v "./.translation_index" \
| sort -n \
| tail -n 1 \
| cut -f1 -d.
)
fi
if [ -z "$timestamp" ]; then
echo 0
else
echo $timestamp
fi
}
# Usage: validProjectName project.name
function validProjectName() {
local PROJECT_NAME=$1
......
typo3/files/templates/languages.txt
View file @ 27d587fa
......@@ -3,7 +3,6 @@ ar
bg
bs
ca
ch
cs
da
de
......@@ -38,7 +37,6 @@ no
pl
pt
pt_BR
qc
ro
ru
sk
......@@ -49,4 +47,5 @@ th
tr
uk
vi
zh
\ No newline at end of file
zh_CN
zh_HK
\ No newline at end of file
typo3/tasks/main.yml
View file @ 27d587fa
---
- name: Create web directory for localisation packages
file: dest=/var/www/{{ domain }}/public/l10n_ter mode=755 state=directory owner=pootle group=www-data recurse=yes
- name: Install git
apt: pkg=git state=installed update_cache=true
......
typo3/templates/scripts/etc/pootle.conf.j2
View file @ 27d587fa
......@@ -8,6 +8,8 @@ POOTLE_LOG=/var/log/pootle/pootle.log
POOTLE_ERR=/var/log/pootle/pootle.err
POOTLE_PO={{ pootle_po_directory }}
STATE_DIR=${POOTLE_SCRIPT_HOME}/.build-language-pack-state
CONF_LANG_COMPAT=${POOTLE_SCRIPT_HOME}/etc/lang-compat.conf
LOCAL_GIT_CLONE_DIRECTORY="{{ pootle_git_directory }}"
......@@ -17,7 +19,7 @@ XSL_LLXML2XLIFF_TARGET=${POOTLE_SCRIPT_HOME}/etc/xsl/xliff2llxml-target.xsl
_LAST_BUILD_FILE="${POOTLE_SCRIPT_HOME}/.build-language-pack-lastbuild"
TER_L10N_PATH=/var/www/{{ domain }}/l10n_ter
TER_L10N_PATH=/var/www/{{ domain }}/public/l10n_ter
# Requires bash >= v4.x
# These are the supported Git branches. Once an older branch
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment