From 087fcf465cf34daad5264b13b5a26b2f102fc88d Mon Sep 17 00:00:00 2001 From: Vishal Bhoj Date: Mon, 24 Nov 2014 12:14:39 +0530 Subject: Add support to build on VPS Change-Id: I5ee21bb42ccee882de3c852014e7a78d52e424c8 Signed-off-by: Vishal Bhoj --- build-scripts/build-android | 1 + build-scripts/helpers | 13 +++++++ node/build | 83 ++++++++++++++++++++++++++++---------------- node/lava-submit | 13 +++++-- node/prepare_build_config.py | 19 +++++++--- 5 files changed, 91 insertions(+), 38 deletions(-) diff --git a/build-scripts/build-android b/build-scripts/build-android index 97bd6fa..6132cac 100644 --- a/build-scripts/build-android +++ b/build-scripts/build-android @@ -6,6 +6,7 @@ # http://www.eclipse.org/legal/epl-v10.html ############################################################################### +export PATH=$PATH:~/bin/ set -o errtrace git config --global user.email "android-build-bot@fake-email.com" diff --git a/build-scripts/helpers b/build-scripts/helpers index ec7f457..a778231 100644 --- a/build-scripts/helpers +++ b/build-scripts/helpers @@ -240,3 +240,16 @@ product2lamc_dev () { echo -n "$1";; esac } + +#Check if we are building on VPS +is_on_ec2() { + [ -e "/etc/cloud/cloud.cfg" ] +} + +get_build_config() { +if is_on_ec2; then + echo "/var/run/" +else + echo "/tmp/" +fi +} diff --git a/node/build b/node/build index 4a068dc..4026433 100755 --- a/node/build +++ b/node/build @@ -12,6 +12,9 @@ # $2 = build CONFIG, base64-encoded # +BUILD_SCRIPT_ROOT=$(readlink -f "$(dirname "${0}")/../build-scripts") + +source "${BUILD_SCRIPT_ROOT}"/helpers REPO_TOOL_URL="https://android.git.linaro.org/gitweb?p=tools/repo.git;a=blob_plain;f=repo;hb=refs/heads/stable" # Dump system release to build log @@ -35,17 +38,19 @@ function update-repo-tool () { mv /tmp/repo /usr/local/bin/repo } -update-repo-tool +if is_on_ec2; then + update-repo-tool +fi + +mkdir -p $(get_build_config)/build-tools -BUILD_SCRIPT_ROOT=$(readlink -f "$(dirname "${0}")/../build-scripts") -mkdir -p /var/run/build-tools if ! $BUILD_SCRIPT_ROOT/../node/prepare_build_config.py --base64 "$2"; then echo "Early exit due to build environment validation failure" exit 1 fi -# At this point, safely sourceable build config is in /var/run/build-tools/build-config -cat /var/run/build-tools/build-config -source /var/run/build-tools/build-config +# At this point, safely sourceable build config is in /tmp/build-tools/build-config +cat $(get_build_config)/build-tools/build-config +source $(get_build_config)/build-tools/build-config if [ -n "$BUILD_CONFIG_REPO" ]; then echo "Fetching build config indirectly from git" @@ -58,7 +63,7 @@ if [ -n "$BUILD_CONFIG_REPO" ]; then git checkout "$BUILD_CONFIG_BRANCH" $BUILD_SCRIPT_ROOT/../node/prepare_build_config.py "$(cat "$BUILD_CONFIG_FILENAME")" cd $save_dir - source /var/run/build-tools/build-config + source $(get_build_config)/build-tools/build-config fi @@ -68,41 +73,59 @@ mount | grep -E "^tmpfs on .+workspace/" | awk ' {print $3}' | xargs --no-run-if umount build/out/target || true umount build/out || true umount build || true -rm -rf build -mkdir build -ramdisk_size=$(get_ramdisk_size $2) +if is_on_ec2; then + rm -rf build +fi +mkdir -p build + +if is_on_ec2; then + ramdisk_size=$(get_ramdisk_size $2) -if [ -z "$ramdisk_size" ]; then - ramdisk_size=0 + if [ -z "$ramdisk_size" ]; then + ramdisk_size=0 + fi fi # Put build/* on a ramdisk to speed up build -if [ -z "$ramdisk_size" -o "$ramdisk_size" != "0" ]; then - mem_total=`cat /proc/meminfo | grep MemTotal | sed -s 's/[^0-9]*\([0-9]*\) kB/\1/'` - if [ "$mem_total" -gt 15680064 ]; then - ramdisk_size=${ramdisk_size:-11750M} - echo "Using $ramdisk_size tmpfs for build" - mount -t tmpfs -o size=$ramdisk_size tmpfs build - elif [ "$mem_total" -gt 12485760 ]; then - # XXX mounting build/out/target on a tmpfs is probably a bit android specific... - ramdisk_size=${ramdisk_size:-11G} - echo "Using $ramdisk_size tmpfs for build/out/target" - mkdir -p build/out/target - mount -t tmpfs -o size=$ramdisk_size tmpfs build/out/target - else - mkdir build/out +if is_on_ec2; then + if [ -z "$ramdisk_size" -o "$ramdisk_size" != "0" ]; then + mem_total=`cat /proc/meminfo | grep MemTotal | sed -s 's/[^0-9]*\([0-9]*\) kB/\1/'` + if [ "$mem_total" -gt 15680064 ]; then + ramdisk_size=${ramdisk_size:-11750M} + echo "Using $ramdisk_size tmpfs for build" + mount -t tmpfs -o size=$ramdisk_size tmpfs build + elif [ "$mem_total" -gt 12485760 ]; then + # XXX mounting build/out/target on a tmpfs is probably a bit android specific... + ramdisk_size=${ramdisk_size:-11G} + echo "Using $ramdisk_size tmpfs for build/out/target#" + mkdir -p build/out/target + mount -t tmpfs -o size=$ramdisk_size tmpfs build/out/target + else + mkdir build/out + fi fi +else + mkdir build/out +fi + +if is_on_ec2; then + chown :nogroup -R /mnt/jenkins + chown jenkins-build:nogroup -R build fi -chown :nogroup -R /mnt/jenkins -chown jenkins-build:nogroup -R build cd build -sudo -E -H -u jenkins-build bash -xes "${BUILD_SCRIPT_ROOT}" "$@" <<\EOF +if is_on_ec2; then + exec_cmd="sudo -E -H -u jenkins-build bash" +else + exec_cmd="bash" +fi +$exec_cmd -xes "${BUILD_SCRIPT_ROOT}" "$@" <<\EOF export BUILD_SCRIPT_ROOT="${1}" HOST="${2}" +source "${BUILD_SCRIPT_ROOT}"/helpers set -a -source /var/run/build-tools/build-config +source $(get_build_config)/build-tools/build-config set +a diff --git a/node/lava-submit b/node/lava-submit index 23a17b3..b06ddd9 100755 --- a/node/lava-submit +++ b/node/lava-submit @@ -9,14 +9,21 @@ ############################################################################### set -xe - BUILD_SCRIPT_ROOT=$(readlink -f "$(dirname "${0}")/../build-scripts") +source "${BUILD_SCRIPT_ROOT}"/helpers cd build -sudo -E -H -u jenkins-build bash -xes "${BUILD_SCRIPT_ROOT}" "$@" <<\EOF + +if is_on_ec2; then + exec_cmd="sudo -E -H -u jenkins-build bash" +else + exec_cmd="bash" +fi +$exec_cmd -xes "${BUILD_SCRIPT_ROOT}" "$@" <<\EOF export BUILD_SCRIPT_ROOT="${1}" set -a -source /var/run/build-tools/build-config +source "${BUILD_SCRIPT_ROOT}"/helpers +source $(get_build_config)/build-tools/build-config set +a if [ -z "$LAVA_SUBMIT" -o "$LAVA_SUBMIT" = "0" ]; then diff --git a/node/prepare_build_config.py b/node/prepare_build_config.py index 4ad5de2..dfcde19 100755 --- a/node/prepare_build_config.py +++ b/node/prepare_build_config.py @@ -6,11 +6,13 @@ import re import pipes import optparse - -SLAVE_TYPE_FILE = "/var/run/build-tools/slave-type" -SLAVE_TYPE_RESTRICTED = "restricted builds" +SLAVE_TYPE_FILE_EC2 = "/var/run/build-tools/slave-type" +# sf-safe build config is written to this file +BUILD_CONFIG_FILE_EC2 = "/var/run/build-tools/build-config" +SLAVE_TYPE_FILE_VPS = "/tmp/build-tools/slave-type" # sf-safe build config is written to this file -BUILD_CONFIG_FILE = "/var/run/build-tools/build-config" +BUILD_CONFIG_FILE_VPS = "/tmp/build-tools/build-config" +SLAVE_TYPE_RESTRICTED = "restricted builds" class BuildConfigMismatchException(Exception): @@ -32,7 +34,10 @@ def shell_unquote(s): def get_slave_type(): slave_type = "" try: - f = open(SLAVE_TYPE_FILE) + if os.path.exists(SLAVE_TYPE_FILE_EC2): + f = open(SLAVE_TYPE_FILE_EC2) + else: + f = open(SLAVE_TYPE_FILE_VPS) slave_type = f.read().rstrip() f.close() except: @@ -101,6 +106,10 @@ def convert_config_to_shell(config_text, out_filename): def main(config_in, is_base64): if is_base64: config_in = base64.b64decode(config_in) + if os.path.exists(BUILD_CONFIG_FILE_EC2): + BUILD_CONFIG_FILE=BUILD_CONFIG_FILE_EC2 + else: + BUILD_CONFIG_FILE=BUILD_CONFIG_FILE_VPS config = convert_config_to_shell(config_in, BUILD_CONFIG_FILE) try: validate_config(config, get_slave_type()) -- cgit v1.2.3