#!/bin/bash set -x usage() { ret=1 [ x"$1" != x ] && ret=$1 echo "MakeRelease: [options]" echo " --abedir - PATH to ABE" echo " --workspace - Specify an alternate workspace" echo " --target XXX - Specify the target triplet to build" echo " --release_name XXX - Specify the release name" echo " --toolchainconfig XXX - [Optional] Explicitly define which toolchain" echo " to build, e.g., 'default', 'gcc5', 'gcc6'. " echo " If not specified, 'default' is the default" echo " --fileserver XXX - Specify the fileserver for tarballs" echo " --manifest XXX - Optionally use a manifest as input" echo " --glibc XXX - Specify Glibc version to build" echo " --gcc XXX - Specify GCC version to build" echo " --binutils XXX - Specify Binutils version to build" echo " --binariesdir XXX - Specify where to upload the generated binaries" echo " --logsdir XXX - Specify where to upload the logs" echo " --canadian - Perform a Canadian-cross build too" echo " --buildnumber XXX - Specify build number" echo " --help" exit $ret } read_var() { local artifact_list=$1 local var=$2 grep "^${var}=" "${artifact_list}" | cut -d = -f 2- } # Jenkins may run only using environment variables, a user needs to always # supply some command line arguments. if test $# -eq 0 -a x"${JENKINS_SERVER_COOKIE}" = x; then usage fi # Set defaults # shellcheck disable=SC2154 if test x"${debug}" = x"true"; then export CONFIG_SHELL="/bin/bash -x" else export CONFIG_SHELL="/bin/bash" fi buildnumber=0 target="" fileserver="" extra= toolchain_config="" user_workspace="${WORKSPACE-/home/${USER-buildslave}/workspace}" abe_dir= binariesdir= logsdir= canadian=false gcc_src= binutils_src= glibc_src= manifest_src= toolchain_config= artifacts_top="releases" manifest_validation=true getopt -o h -l target:,release_name:,fileserver:,workspace:,toolchainconfig:,manifest:,glibc:,gcc:,binutils:,help,abedir:,binariesdir:,logsdir:,canadian,buildnumber:,artifacts_top:,manifest_validation: -Q while test $# -gt 0; do case $1 in --abedir) abe_dir=$2 ; shift ;; --workspace) user_workspace=$2 ; shift ;; --target) target=$2 ; shift ;; --release_name) release_name=$2 ; shift ;; --toolchainconfig) toolchain_config=$2 ; shift ;; --fileserver) fileserver=$2 ; shift ;; --manifest) manifest_src=$2 ; shift ;; --glibc) glibc_src=$2 ; shift ;; --gcc) gcc_src=$2 ; shift ;; --binutils) binutils_src=$2 ; shift ;; --binariesdir) binariesdir=$2 ; shift ;; --logsdir) logsdir=$2 ; shift ;; --canadian) canadian=true ;; --buildnumber) buildnumber=$2 ; shift ;; --artifacts_top) artifacts_top=$2; shift ;; --manifest_validation) manifest_validation=$2; shift ;; -h|--help) usage 0 ;; --) break ;; *) usage ;; esac shift done if test x"${abe_dir}" = x; then echo "Error: --abedir missing" usage fi if test x"${fileserver}" = x; then rsh="sh -c" rsync_flags="" else # Hardcode port 22 for the time being. We use this parameter to # upload release tarballs to dev-01, but our ssh default config # points to the host container where the directory with binary # tarballs is not mapped. rsh="ssh -p 22 $fileserver" rsync_flags="ssh -p 22" fi user_snapshots="${user_workspace}/snapshots" if test -e ${user_workspace}; then cat << EOF > ${user_workspace}/BUILD-INFO.txt Format-Version: 0.5 Files-Pattern: * License-Type: open EOF fi # Create a build directory if test ! -d ${user_workspace}/_build; then mkdir -p ${user_workspace}/_build else rm -fr ${user_workspace}/_build/* fi # By default, always update all sources update="" if test x"${toolchain_config}" != x"" \ -a x"${toolchain_config}" != x"default"; then extra="${extra} --extraconfigdir $abe_dir/config/${toolchain_config}" fi if [ x"$target" = x"aarch64-linux-gnu_ilp32" ]; then extra="$extra --extraconfigdir $abe_dir/config/ilp32-dev" fi # Use the newly created build directory cd ${user_workspace}/_build || exit if ! test -e host.conf; then $CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} \ --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref fi if test x"${release_name}" != x; then release="${release_name}" else echo "MakeRelease.job must be invoked with a --release_name string." exit 1 fi manifest=${manifest_src:+--manifest ${manifest_src}} glibc=${glibc_src:+glibc=${glibc_src}} binutils=${binutils_src:+binutils=${binutils_src}} gcc=${gcc_src:+gcc=${gcc_src}} srcs="${gcc} ${binutils} ${glibc} ${manifest}" logfile=${user_workspace}/MakeRelease-${buildnumber}-$(uname -m).log # Set ABE's --target setting. No setting means native. # If manifest file is set, then ABE must use its setting. target_opt= if [ x"${target}" != x"native" ] && [ x"${target}" != x ] && \ [ x"$manifest" = x"" ]; then target_opt="--target ${target}" fi # Build a binary release tarball # Remove logfile if present (for some unknown reason) rm -f ${logfile} abe_ret=0 manifests=() tarballs=() # Canadian cross builds require a Linux hosted cross compiler first if test x"${canadian}" = x"true"; then $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} $target_opt --build all ${extra} --tarbin >> ${logfile} abe_ret=$? host="--host i686-w64-mingw32" manifests+=( "$(read_var ${user_workspace}/artifacts1.txt manifest)" ) else host="" fi # If we're not building the mingw32 compiler 'abe_ret' will be zero and the # following conditional will build the linux cross-compiler. Otherwise it'll # build the mingw32 compiler only if the previous cross-compiler build was # successful. if test ${abe_ret} -eq 0; then $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} $target_opt ${host} --build all ${extra} >> ${logfile} abe_ret=$? manifests+=( "$(read_var ${user_workspace}/artifacts2.txt manifest)" ) fi if test x"${canadian}" = x"true"; then # take the toolchain, sysroot and runtime tarballs from the linux build mapfile -t -O "${#tarballs[@]}" tarballs < <( \ read_var ${user_workspace}/artifacts1.txt '\(toolchain\|sysroot\|runtime\)\(_asc\)\?' ) # and just the toolchain from the mingw build mapfile -t -O "${#tarballs[@]}" tarballs < <( \ read_var ${user_workspace}/artifacts2.txt 'toolchain\(_asc\)\?' ) else mapfile -t -O "${#tarballs[@]}" tarballs < <( \ read_var ${user_workspace}/artifacts2.txt '\(toolchain\|sysroot\|runtime\)\(_asc\)\?' ) fi # MakeRelease.job doesn't require an input gcc_src parameter (it'll invoke ABE # with the default) so we can't rely on it for the gcc version. Parse the # generated manifest file name instead, e.g., # gcc-linaro-6.1.1-2016.08-rc1-linux-manifest.txt # gcc-linaro-6.1.1-2016.08-rc1-win32-manifest.txt manifest_match="*-manifest.txt" # Make sure we have at least one manifest file found. if test -z "${manifests[0]}"; then echo "Couldn't find a manifest file for the recent build." exit 1 fi # Major.Minor.Point gcc_full_version=`basename ${manifests[0]} | awk -F '-' '{ print $3 }'` # Before GCC 5 the GCC release name was Major.Minor.Point. With GCC 5 # and later the GCC release name is Major.Minor. gcc_major=`echo $gcc_full_version | awk -F '.' '{ print $1 }'` gcc_minor=`echo $gcc_full_version | awk -F '.' '{ print $2 }'` gcc_release=${gcc_major}.${gcc_minor} if test ${gcc_major} -lt 5; then gcc_point=`echo $gcc_full_version | awk -F '.' '{ print $3 }'` gcc_release=${gcc_major}.${gcc_minor}.${gcc_point} fi # Strip off everything but the DATE[-SPIN][-RC], complicated by the fact # that the spin and rc are both optional and not mutually exclusive. series_date=${manifests[0]##*-${gcc_full_version}-} series_date=${series_date%${manifest_match}} # We aren't sure if we're getting the linux or win32 manifest so strip both. series_date=${series_date%-linux} series_date=${series_date%-win32} # Fileserver location of binary tarballs and build logfiles binaries="$artifacts_top/binaries/${gcc_release}-${series_date}-${buildnumber}/${target}" logs="$artifacts_top/logs/${gcc_release}-${series_date}-${buildnumber}/${target}" if [ "x${binariesdir}" != "x" ] && \ [ "x${binariesdir}" != "x${binaries}" ]; then echo "WARNING: Provided binaries dirname does not match with what was computed" echo "Provided: ${binariesdir}" echo "Computed: ${binaries}" binaries=${binariesdir} fi if [ "x${logsdir}" != "x" ] && \ [ "x${logsdir}" != "x${logs}" ]; then echo "WARNING: Provided logs dirname does not match with what was computed" echo "Provided: ${logsdir}" echo "Computed: ${logs}" logs=${logsdir} fi # Copy the build log to the fileserver if test -e ${logfile}; then echo "Compressing log file..." xz ${logfile} $rsh "if test ! -d ${logs}; then mkdir -p ${logs}; fi" echo "Copying compressed log file ${logfile}.xz to ${fileserver:+$fileserver:}${logs}..." rsync -a ${rsync_flags:+-e "$rsync_flags"} ${logfile}.xz ${fileserver:+$fileserver:}${logs}/ fi # force a failure if abe has build problems. if test ${abe_ret} -ne 0; then exit 1 fi # Copy the binaries to the fileserver $rsh "if test ! -d ${binaries}; then mkdir -p ${binaries}; fi" # There could be a linux and/or a win32 manifest. for manifest in "${manifests[@]}" do rsync -a ${rsync_flags:+-e "$rsync_flags"} ${manifest} ${fileserver:+$fileserver:}${binaries}/ done rsync -a ${rsync_flags:+-e "$rsync_flags"} "${tarballs[@]}" ${fileserver:+$fileserver:}${binaries}/ if $manifest_validation; then echo "Starting manifest validation." git clone --depth 1 git://git.linaro.org/toolchain/abe-tests.git for manifest in "${manifests[@]}"; do ./abe-tests/validate-manifest.pl "${manifest}" if test $? -ne 0; then echo "Manifest validation failed." exit 1 else echo "Manifest validation passed." fi done fi exit 0