Commit 80524dc2 authored by Christian Kuhn's avatar Christian Kuhn
Browse files

[TASK] First baseimage and php72 refactoring

parent d804c5f9
NAME_BASEIMAGE = typo3gmbh/baseimage
MAJOR_BASEIMAGE=2
MAJOR_BASEIMAGE=3
MINOR_BASEIMAGE=0
PATCHLEVEL_BASEIMAGE=4
PATCHLEVEL_BASEIMAGE=0
FULLVERSION_BASEIMAGE=$(MAJOR_BASEIMAGE).$(MINOR_BASEIMAGE).$(PATCHLEVEL_BASEIMAGE)
SHORTVERSION_BASEIMAGE=$(MAJOR_BASEIMAGE).$(MINOR_BASEIMAGE)
......@@ -49,9 +49,9 @@ FULLVERSION_PHP71=$(MAJOR_PHP71).$(MINOR_PHP71).$(PATCHLEVEL_PHP71)
SHORTVERSION_PHP71=$(MAJOR_PHP71).$(MINOR_PHP71)
NAME_PHP72 = typo3gmbh/php72
MAJOR_PHP72=2
MAJOR_PHP72=3
MINOR_PHP72=0
PATCHLEVEL_PHP72=4
PATCHLEVEL_PHP72=0
FULLVERSION_PHP72=$(MAJOR_PHP72).$(MINOR_PHP72).$(PATCHLEVEL_PHP72)
SHORTVERSION_PHP72=$(MAJOR_PHP72).$(MINOR_PHP72)
......@@ -99,9 +99,9 @@ FULLVERSION_BAMBOO_PHP71=$(MAJOR_BAMBOO_PHP71).$(MINOR_BAMBOO_PHP71).$(PATCHLEVE
SHORTVERSION_BAMBOO_PHP71=$(MAJOR_BAMBOO_PHP71).$(MINOR_BAMBOO_PHP71)
NAME_BAMBOO_PHP72 = typo3gmbh/bamboo-remote-agent-php72
MAJOR_BAMBOO_PHP72=2
MAJOR_BAMBOO_PHP72=3
MINOR_BAMBOO_PHP72=0
PATCHLEVEL_BAMBOO_PHP72=4
PATCHLEVEL_BAMBOO_PHP72=0
FULLVERSION_BAMBOO_PHP72=$(MAJOR_BAMBOO_PHP72).$(MINOR_BAMBOO_PHP72).$(PATCHLEVEL_BAMBOO_PHP72)
SHORTVERSION_BAMBOO_PHP72=$(MAJOR_BAMBOO_PHP72).$(MINOR_BAMBOO_PHP72)
......@@ -458,7 +458,7 @@ clean_images_php72:
build_bamboo_php53: build_php53
rm -rf build_bamboo-php53
cp -pR bamboo-remote-agent-php53 build_bamboo-php53
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP53) >> build_bamboo-php53/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP53) >> build_bamboo-php53/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP53):$(FULLVERSION_BAMBOO_PHP53) build_bamboo-php53
release_bamboo_php53:
......@@ -483,7 +483,7 @@ clean_images_bamboo_php53:
build_bamboo_php54: build_php54
rm -rf build_bamboo-php54
cp -pR bamboo-remote-agent-php54 build_bamboo-php54
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP54) >> build_bamboo-php54/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP54) >> build_bamboo-php54/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP54):$(FULLVERSION_BAMBOO_PHP54) build_bamboo-php54
release_bamboo_php54:
......@@ -508,7 +508,7 @@ clean_images_bamboo_php54:
build_bamboo_php55: build_php55
rm -rf build_bamboo-php55
cp -pR bamboo-remote-agent-php55 build_bamboo-php55
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP55) >> build_bamboo-php55/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP55) >> build_bamboo-php55/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP55):$(FULLVERSION_BAMBOO_PHP55) build_bamboo-php55
release_bamboo_php55:
......@@ -533,7 +533,7 @@ clean_images_bamboo_php55:
build_bamboo_php56: build_php56
rm -rf build_bamboo-php56
cp -pR bamboo-remote-agent-php56 build_bamboo-php56
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP56) >> build_bamboo-php56/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP56) >> build_bamboo-php56/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP56):$(FULLVERSION_BAMBOO_PHP56) build_bamboo-php56
release_bamboo_php56:
......@@ -558,7 +558,7 @@ clean_images_bamboo_php56:
build_bamboo_php70: build_php70
rm -rf build_bamboo-php70
cp -pR bamboo-remote-agent-php70 build_bamboo-php70
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP70) >> build_bamboo-php70/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP70) >> build_bamboo-php70/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP70):$(FULLVERSION_BAMBOO_PHP70) build_bamboo-php70
release_bamboo_php70:
......@@ -583,7 +583,7 @@ clean_images_bamboo_php70:
build_bamboo_php71: build_php71
rm -rf build_bamboo-php71
cp -pR bamboo-remote-agent-php71 build_bamboo-php71
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP71) >> build_bamboo-php71/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP71) >> build_bamboo-php71/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP71):$(FULLVERSION_BAMBOO_PHP71) build_bamboo-php71
release_bamboo_php71:
......@@ -608,7 +608,7 @@ clean_images_bamboo_php71:
build_bamboo_php72: build_php72
rm -rf build_bamboo-php72
cp -pR bamboo-remote-agent-php72 build_bamboo-php72
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP72) >> build_bamboo-php72/config/bamboo/bamboo-capabilities.properties
echo system.imageVersion=$(FULLVERSION_BAMBOO_PHP72) >> build_bamboo-php72/config/bamboo-capabilities.properties
docker build -t $(NAME_BAMBOO_PHP72):$(FULLVERSION_BAMBOO_PHP72) build_bamboo-php72
release_bamboo_php72:
......
......@@ -2,5 +2,6 @@ FROM typo3gmbh/php72:2.0
MAINTAINER TYPO3 GmbH <info@typo3.com>
ADD . /pd_build
RUN /pd_build/install.sh
CMD ["/sbin/my_init"]
RUN /pd_build/bamboo-agent.sh && \
/pd_build/finalize.sh
......@@ -3,19 +3,13 @@ set -e
source /pd_build/buildconfig
set -x
# Create bamboo work directory
mkdir -p /srv/bamboo-agent-home
chmod 0775 /srv/bamboo-agent-home
chown bamboo:bamboo /srv/bamboo-agent-home
# We don't know which user will execute bamboo, we have to use 777 here
mkdir -p /srv/bamboo/bin
chmod 0777 /srv/bamboo
chmod 0777 /srv/bamboo/bin
# Install bamboo remote agent
curl -SL --progress-bar https://bamboo.typo3.com/agentServer/agentInstaller/ -o /tmp/bamboo-installer.jar
/usr/bin/java -Dbamboo.home=/srv/bamboo-agent-home -jar /tmp/bamboo-installer.jar https://bamboo.typo3.com/agentServer install
chown -R bamboo:bamboo /srv/bamboo-agent-home
rm -f /tmp/bamboo-installer.jar
# Download bamboo remote agent
curl -SL --progress-bar https://bamboo.typo3.com/agentServer/agentInstaller/ -o /srv/bamboo-installer.jar
## Configure properties
cp -a /pd_build/config/bamboo/bamboo-capabilities.properties /srv/bamboo-agent-home/bin
## Enable agent
cp -a /pd_build/runit/bamboo-agent /etc/service/bamboo-agent
# Configure properties
cp -a /pd_build/config/bamboo-capabilities.properties /srv/bamboo/bin
\ No newline at end of file
......@@ -3,9 +3,9 @@ export DEBIAN_FRONTEND=noninteractive
function minimal_apt_get_install()
{
if [[ ! -e /var/lib/apt/lists/lock ]]; then
apt-get update
fi
apt-get install -y --no-install-recommends "$@"
if [[ ! -e /var/lib/apt/lists/lock ]]; then
apt-get update
fi
apt-get install -y --no-install-recommends "$@"
}
......@@ -6,9 +6,9 @@ set -x
apt-get clean
rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/* \
/usr/local/src/* \
#
/tmp/* \
/var/tmp/* \
/usr/local/src/* \
#
rm -rf /pd_build
#!/bin/bash
set -e
source /pd_build/buildconfig
set -x
/pd_build/prepare.sh
/pd_build/bamboo-agent.sh
/pd_build/finalize.sh
#!/bin/bash
set -e
source /pd_build/buildconfig
set -x
#!/bin/sh
mkdir -p /var/log/bamboo-agent
exec svlogd -tt /var/log/bamboo-agent
#!/bin/sh
exec 2>&1
exec /sbin/setuser bamboo /srv/bamboo-agent-home/bin/bamboo-agent.sh console
......@@ -4,8 +4,5 @@ MAINTAINER TYPO3 GmbH <info@typo3.com>
ADD . /bd_build
RUN /bd_build/prepare.sh && \
/bd_build/system_services.sh && \
/bd_build/utilities.sh && \
/bd_build/cleanup.sh
CMD ["/sbin/my_init"]
/bd_build/utilities.sh && \
/bd_build/cleanup.sh
#!/usr/bin/python3 -u
import os, os.path, sys, stat, signal, errno, argparse, time, json, re
KILL_PROCESS_TIMEOUT = 5
KILL_ALL_PROCESSES_TIMEOUT = 5
LOG_LEVEL_ERROR = 1
LOG_LEVEL_WARN = 1
LOG_LEVEL_INFO = 2
LOG_LEVEL_DEBUG = 3
SHENV_NAME_WHITELIST_REGEX = re.compile('[^\w\-_\.]')
log_level = None
terminated_child_processes = {}
class AlarmException(Exception):
pass
def error(message):
if log_level >= LOG_LEVEL_ERROR:
sys.stderr.write("*** %s\n" % message)
def warn(message):
if log_level >= LOG_LEVEL_WARN:
sys.stderr.write("*** %s\n" % message)
def info(message):
if log_level >= LOG_LEVEL_INFO:
sys.stderr.write("*** %s\n" % message)
def debug(message):
if log_level >= LOG_LEVEL_DEBUG:
sys.stderr.write("*** %s\n" % message)
def ignore_signals_and_raise_keyboard_interrupt(signame):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
raise KeyboardInterrupt(signame)
def raise_alarm_exception():
raise AlarmException('Alarm')
def listdir(path):
try:
result = os.stat(path)
except OSError:
return []
if stat.S_ISDIR(result.st_mode):
return sorted(os.listdir(path))
else:
return []
def is_exe(path):
try:
return os.path.isfile(path) and os.access(path, os.X_OK)
except OSError:
return False
def import_envvars(clear_existing_environment = True, override_existing_environment = True):
if not os.path.exists("/etc/container_environment"):
return
new_env = {}
for envfile in listdir("/etc/container_environment"):
name = os.path.basename(envfile)
with open("/etc/container_environment/" + envfile, "r") as f:
# Text files often end with a trailing newline, which we
# don't want to include in the env variable value. See
# https://github.com/phusion/baseimage-docker/pull/49
value = re.sub('\n\Z', '', f.read())
new_env[name] = value
if clear_existing_environment:
os.environ.clear()
for name, value in new_env.items():
if override_existing_environment or not name in os.environ:
os.environ[name] = value
def export_envvars(to_dir = True):
if not os.path.exists("/etc/container_environment"):
return
shell_dump = ""
for name, value in os.environ.items():
if name in ['HOME', 'USER', 'GROUP', 'UID', 'GID', 'SHELL']:
continue
if to_dir:
with open("/etc/container_environment/" + name, "w") as f:
f.write(value)
shell_dump += "export " + sanitize_shenvname(name) + "=" + shquote(value) + "\n"
with open("/etc/container_environment.sh", "w") as f:
f.write(shell_dump)
with open("/etc/container_environment.json", "w") as f:
f.write(json.dumps(dict(os.environ)))
_find_unsafe = re.compile(r'[^\w@%+=:,./-]').search
def shquote(s):
"""Return a shell-escaped version of the string *s*."""
if not s:
return "''"
if _find_unsafe(s) is None:
return s
# use single quotes, and put single quotes into double quotes
# the string $'b is then quoted as '$'"'"'b'
return "'" + s.replace("'", "'\"'\"'") + "'"
def sanitize_shenvname(s):
return re.sub(SHENV_NAME_WHITELIST_REGEX, "_", s)
# Waits for the child process with the given PID, while at the same time
# reaping any other child processes that have exited (e.g. adopted child
# processes that have terminated).
def waitpid_reap_other_children(pid):
global terminated_child_processes
status = terminated_child_processes.get(pid)
if status:
# A previous call to waitpid_reap_other_children(),
# with an argument not equal to the current argument,
# already waited for this process. Return the status
# that was obtained back then.
del terminated_child_processes[pid]
return status
done = False
status = None
while not done:
try:
# https://github.com/phusion/baseimage-docker/issues/151#issuecomment-92660569
this_pid, status = os.waitpid(pid, os.WNOHANG)
if this_pid == 0:
this_pid, status = os.waitpid(-1, 0)
if this_pid == pid:
done = True
else:
# Save status for later.
terminated_child_processes[this_pid] = status
except OSError as e:
if e.errno == errno.ECHILD or e.errno == errno.ESRCH:
return None
else:
raise
return status
def stop_child_process(name, pid, signo = signal.SIGTERM, time_limit = KILL_PROCESS_TIMEOUT):
info("Shutting down %s (PID %d)..." % (name, pid))
try:
os.kill(pid, signo)
except OSError:
pass
signal.alarm(time_limit)
try:
try:
waitpid_reap_other_children(pid)
except OSError:
pass
except AlarmException:
warn("%s (PID %d) did not shut down in time. Forcing it to exit." % (name, pid))
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
try:
waitpid_reap_other_children(pid)
except OSError:
pass
finally:
signal.alarm(0)
def run_command_killable(*argv):
filename = argv[0]
status = None
pid = os.spawnvp(os.P_NOWAIT, filename, argv)
try:
status = waitpid_reap_other_children(pid)
except BaseException as s:
warn("An error occurred. Aborting.")
stop_child_process(filename, pid)
raise
if status != 0:
if status is None:
error("%s exited with unknown status\n" % filename)
else:
error("%s failed with status %d\n" % (filename, os.WEXITSTATUS(status)))
sys.exit(1)
def run_command_killable_and_import_envvars(*argv):
run_command_killable(*argv)
import_envvars()
export_envvars(False)
def kill_all_processes(time_limit):
info("Killing all processes...")
try:
os.kill(-1, signal.SIGTERM)
except OSError:
pass
signal.alarm(time_limit)
try:
# Wait until no more child processes exist.
done = False
while not done:
try:
os.waitpid(-1, 0)
except OSError as e:
if e.errno == errno.ECHILD:
done = True
else:
raise
except AlarmException:
warn("Not all processes have exited in time. Forcing them to exit.")
try:
os.kill(-1, signal.SIGKILL)
except OSError:
pass
finally:
signal.alarm(0)
def run_startup_files():
# Run /etc/my_init.d/*
for name in listdir("/etc/my_init.d"):
filename = "/etc/my_init.d/" + name
if is_exe(filename):
info("Running %s..." % filename)
run_command_killable_and_import_envvars(filename)
# Run /etc/rc.local.
if is_exe("/etc/rc.local"):
info("Running /etc/rc.local...")
run_command_killable_and_import_envvars("/etc/rc.local")
def start_runit():
info("Booting runit daemon...")
pid = os.spawnl(os.P_NOWAIT, "/usr/bin/runsvdir", "/usr/bin/runsvdir",
"-P", "/etc/service")
info("Runit started as PID %d" % pid)
return pid
def wait_for_runit_or_interrupt(pid):
try:
status = waitpid_reap_other_children(pid)
return (True, status)
except KeyboardInterrupt:
return (False, None)
def shutdown_runit_services():
debug("Begin shutting down runit services...")
os.system("/usr/bin/sv down /etc/service/*")
def wait_for_runit_services():
debug("Waiting for runit services to exit...")
done = False
while not done:
done = os.system("/usr/bin/sv status /etc/service/* | grep -q '^run:'") != 0
if not done:
time.sleep(0.1)
def main(args):
import_envvars(False, False)
export_envvars()
if not args.skip_startup_files:
run_startup_files()
runit_exited = False
exit_code = None
if not args.skip_runit:
runit_pid = start_runit()
try:
exit_status = None
if len(args.main_command) == 0:
runit_exited, exit_code = wait_for_runit_or_interrupt(runit_pid)
if runit_exited:
if exit_code is None:
info("Runit exited with unknown status")
exit_status = 1
else:
exit_status = os.WEXITSTATUS(exit_code)
info("Runit exited with status %d" % exit_status)
else:
info("Running %s..." % " ".join(args.main_command))
pid = os.spawnvp(os.P_NOWAIT, args.main_command[0], args.main_command)
try:
exit_code = waitpid_reap_other_children(pid)
if exit_code is None:
info("%s exited with unknown status." % args.main_command[0])
exit_status = 1
else:
exit_status = os.WEXITSTATUS(exit_code)
info("%s exited with status %d." % (args.main_command[0], exit_status))
except KeyboardInterrupt:
stop_child_process(args.main_command[0], pid)
raise
except BaseException as s:
warn("An error occurred. Aborting.")
stop_child_process(args.main_command[0], pid)
raise
sys.exit(exit_status)
finally:
if not args.skip_runit:
shutdown_runit_services()
if not runit_exited:
stop_child_process("runit daemon", runit_pid)
wait_for_runit_services()
# Parse options.
parser = argparse.ArgumentParser(description = 'Initialize the system.')
parser.add_argument('main_command', metavar = 'MAIN_COMMAND', type = str, nargs = '*',
help = 'The main command to run. (default: runit)')
parser.add_argument('--skip-startup-files', dest = 'skip_startup_files',
action = 'store_const', const = True, default = False,
help = 'Skip running /etc/my_init.d/* and /etc/rc.local')
parser.add_argument('--skip-runit', dest = 'skip_runit',
action = 'store_const', const = True, default = False,
help = 'Do not run runit services')
parser.add_argument('--no-kill-all-on-exit', dest = 'kill_all_on_exit',
action = 'store_const', const = False, default = True,
help = 'Don\'t kill all processes on the system upon exiting')
parser.add_argument('--quiet', dest = 'log_level',
action = 'store_const', const = LOG_LEVEL_WARN, default = LOG_LEVEL_INFO,
help = 'Only print warnings and errors')
args = parser.parse_args()
log_level = args.log_level
if args.skip_runit and len(args.main_command) == 0:
error("When --skip-runit is given, you must also pass a main command.")
sys.exit(1)
# Run main function.
signal.signal(signal.SIGTERM, lambda signum, frame: ignore_signals_and_raise_keyboard_interrupt('SIGTERM'))
signal.signal(signal.SIGINT, lambda signum, frame: ignore_signals_and_raise_keyboard_interrupt('SIGINT'))
signal.signal(signal.SIGALRM, lambda signum, frame: raise_alarm_exception())
try:
main(args)
except KeyboardInterrupt:
warn("Init system aborted.")
exit(2)
finally:
if args.kill_all_on_exit:
kill_all_processes(KILL_ALL_PROCESSES_TIMEOUT)
#!/usr/bin/python3
import sys, os, pwd
if len(sys.argv) < 3:
sys.stderr.write("Usage: /sbin/setuser USERNAME COMMAND [args..]\n")
sys.exit(1)
def abort(message):
sys.stderr.write("setuser: %s\n" % message)
sys.exit(1)
username = sys.argv[1]
try:
user = pwd.getpwnam(username)
except KeyError:
abort("user %s not found" % username)
os.initgroups(username, user.pw_gid)
os.setgid(user.pw_gid)
os.setuid(user.pw_uid)
os.environ['USER'] = username
os.environ['HOME'] = user.pw_dir
os.environ['UID'] = str(user.pw_uid)
try:
os.execvp(sys.argv[2], sys.argv[2:])
except OSError as e:
abort("cannot execute %s: %s" % (sys.argv[2], str(e)))
export LC_ALL=C
export DEBIAN_FRONTEND=noninteractive
minimal_apt_get_install='apt-get install -y --no-install-recommends'
# Default services
# Set 1 to the service you want to disable
export DISABLE_SYSLOG=${DISABLE_SYSLOG:-0}
export DISABLE_CRON=${DISABLE_CRON:-0}
......@@ -3,45 +3,45 @@ set -e
source /bd_build/buildconfig
set -x
## Temporarily disable dpkg fsync to make building faster.
# Temporarily disable dpkg fsync to make building faster.
if [[ ! -e /etc/dpkg/dpkg.cfg.d/docker-apt-speedup ]]; then
echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup
fi
## Prevent initramfs updates from trying to run grub and lilo.
## https://journal.paul.querna.org/articles/2013/10/15/docker-ubuntu-on-rackspace/
## http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594189
# Prevent initramfs updates from trying to run grub and lilo.
# https://journal.paul.querna.org/articles/2013/10/15/docker-ubuntu-on-rackspace/
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594189
export INITRD=no
mkdir -p /etc/container_environment
echo -n no > /etc/container_environment/INITRD
## Enable Ubuntu Universe and Multiverse.
# Enable Ubuntu Universe and Multiverse.
sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
sed -i 's/^#\s*\(deb.*multiverse\)$/\1/g' /etc/apt/sources.list
apt-get update
## Fix some issues with APT packages.
## See https://github.com/dotcloud/docker/issues/1024
# Fix some issues with APT packages.
# See https://github.com/dotcloud/docker/issues/1024
dpkg-divert --local --rename --add /sbin/initctl
ln -sf /bin/true /sbin/initctl
## Replace the 'ischroot' tool to make it always return true.
## Prevent initscripts updates from breaking /dev/shm.
## https://journal.paul.querna.org/articles/2013/10/15/docker-ubuntu-on-rackspace/
## https://bugs.launchpad.net/launchpad/+bug/974584
# Replace the 'ischroot' tool to make it always return true.
# Prevent initscripts updates from breaking /dev/shm.
# https://journal.paul.querna.org/articles/2013/10/15/docker-ubuntu-on-rackspace/
# https://bugs.launchpad.net/launchpad/+bug/974584
dpkg-divert --local --rename --add /usr/bin/ischroot