#!/usr/bin/env bash # This script is an experiment on how to build LLVM in Jenkins. # The steps here are based on the scripts currently in: # https://git.linaro.org/toolchain/llvm/linaro-scripts.git set -exu set -o pipefail BASEDIR=$(dirname "$(readlink -f "$0")") # shellcheck source=tcwg-llvm-common.sh . "$BASEDIR/tcwg-llvm-common.sh" # Syntax SYN_WORKSPACE="--workspace=/path/to/workspace" SYN_REVISION="--revision=rNNNNN (SVN revision, def. tip)" SYN_BUILDTYPE="--buildtype={Release|Debug} (def. Release)" SYN_ASSERTS="--asserts={True|False} (def. True)" SYN_BUILDJOBS="--buildjobs=N (def. CPUS)" SYN_LINKJOBS="--linkjobs=N (def. RAM+1)" SYN_TOOLCHAIN="--toolchain=http://url/for/tarball" SYN_CMAKEFLAGS="--cmakeflags='cmake flags ...'" SYN_BUILDID="--buildid=" SYN_TOOLCHAIN_FILE="--toolchain-file= (file name to copy the produced toolchain file name)" SYN_COMPILER="--compiler=clang|gcc" SYN_REPOSITORY="--repository=http://url/for/git/repo/" SYN_BRANCH="--branch=branchname" SYN_PROJECTS="--projects='llvm[@sha1] clang[@sha1] etc'" SYNTAX="$0 $SYN_WORKSPACE $SYN_REVISION $SYN_ASSERTS $SYN_LINKJOBS $SYN_BUILDJOBS $SYN_TOOLCHAIN $SYN_BUILDID $SYN_TOOLCHAIN_FILE $SYN_CMAKEFLAGS $SYN_COMPILER $SYN_REPOSITORY $SYN_BRANCH $SYN_PROJECTS" # Environment Variables and default values WORKSPACE=$(pwd) REVISION= BUILDTYPE="Release" ASSERTS="True" SRCDIR="" BUILDDIR="" INSTDIR="" BUILDJOBS=$CPUS TOOLCHAIN="" TOOLCHAIN_FILE="" CMAKEFLAGS="" BUILDID="" REPOSITORY="https://git-us.linaro.org/toolchain/llvm" BRANCH="master" PROJECTS="" # Command line parsing while [ "$#" -gt 0 ]; do ARG=$(echo "$1" | cut -d "=" -f 1) VAL=$(echo "$1" | cut -s -d "=" -f 2-) case "$ARG" in --workspace) if [ -d "$VAL" ]; then WORKSPACE="$VAL" else echo "ERROR: $1" echo "Syntax: $SYN_WORKSPACE" exit 1 fi shift ;; --revision) if [ ! -z "$VAL" ]; then if echo "$VAL" | grep -q "r[0-9]\+"; then REVISION="$VAL" else echo "ERROR: $1" echo "Syntax: $SYN_REVISION" exit 1 fi fi shift ;; --buildid) if [ "$VAL" != "" ]; then BUILDID="$VAL" else echo "ERROR: $1" echo "Syntax: $SYN_BUILDID" exit 1 fi shift ;; --buildtype) if [ "$VAL" = "Debug" ]; then BUILDTYPE="$VAL" elif [ "$VAL" != "Release" ]; then echo "ERROR: $1" echo "Syntax: $SYN_BUILDTYPE" exit 1 fi shift ;; --asserts) if [ "$VAL" = "False" ]; then ASSERTS="$VAL" elif [ "$VAL" != "True" ]; then echo "ERROR: $1" echo "Syntax: $SYN_ASSERTS" exit 1 fi shift ;; --buildjobs) if [ "$VAL" -gt 0 ]; then BUILDJOBS="$VAL" else echo "ERROR: Build jobs '$VAL' not valid" echo "Syntax: $SYN_BUILDJOBS" exit 1 fi shift ;; --linkjobs) if [ "$VAL" -gt 0 ]; then LINKJOBS="$VAL" else echo "ERROR: Link jobs '$VAL' not valid" echo "Syntax: $SYN_LINKJOBS" exit 1 fi shift ;; --repository) if [ ! -z "$VAL" ]; then REPOSITORY="$VAL" else echo "ERROR: Git repository '$VAL' not valid" echo "Syntax: $SYN_REPOSITORY" exit 1 fi shift ;; --branch) if [ ! -z "$VAL" ]; then BRANCH="$VAL" else echo "ERROR: Git branch '$VAL' not valid" echo "Syntax: $SYN_BRANCH" exit 1 fi shift ;; --toolchain) download_toolchain "$WORKSPACE" "$VAL" shift ;; --compiler) set_compiler "$VAL" shift ;; --toolchain-file) if [ "$VAL" != "" ]; then TOOLCHAIN_FILE="$VAL" else echo "ERROR: Toolchain file '$VAL' not valid" echo "Syntax: $SYN_TOOLCHAIN_FILE" exit 1 fi shift ;; --cmakeflags) CMAKEFLAGS="$VAL" shift ;; --projects) PROJECTS="$VAL" shift ;; *) echo "ERROR: Invalid argument '$1'" echo "Syntax: $SYNTAX" exit 1 ;; esac done # Validate options if [ "$BUILDID" = "" ] && [ "$REVISION" = "" ]; then echo "ERROR: Missing build ID and SVN revision" echo "$SYNTAX" exit 1 fi if [ "$WORKSPACE" = "" ] || [ ! -d "$WORKSPACE" ]; then echo "ERROR: Missing workspace" echo "$SYNTAX" exit 1 fi if [ "$BUILDTYPE" = "" ] || [ "$ASSERTS" = "" ]; then echo "ERROR: Missing build type / asserts option" echo "$SYNTAX" exit 1 fi if [ "$LINKJOBS" = "" ] || [ "$BUILDJOBS" = "" ]; then echo "ERROR: Missing number of build / link jobs" echo "$SYNTAX" exit 1 fi # Dirs SRCDIR="$WORKSPACE/llvm" BUILDDIR="$WORKSPACE/build" INSTDIR="$WORKSPACE/install" REFDIR="$HOME/llvm-reference" # Dump echo "REVISION = $REVISION" echo "BUILDTYPE = $BUILDTYPE" echo "ASSERTS = $ASSERTS" echo "SRCDIR = $SRCDIR" echo "BUILDDIR = $BUILDDIR" echo "INSTDIR = $INSTDIR" echo "REFDIR = $REFDIR" echo "BUILDJOBS = $BUILDJOBS" echo "LINKJOBS = $LINKJOBS" echo "TOOLCHAIN = $TOOLCHAIN" echo "CC = $CC" echo "CXX = $CXX" echo "CMAKEFLAGS = $CMAKEFLAGS" echo "BRANCH = $BRANCH" echo "PROJECTS = $PROJECTS" # Logs LOGBASE="$WORKSPACE/tcwg-llvm" LOGEXT="txt" # Checkout LLVM_REVS="" CLONE_STATUS=0 { flock -s 9 for proj_ref in $LLVM_PROJECTS; do proj="$(echo "$proj_ref" | cut -d ":" -f 1)" dir="$(echo "$proj_ref" | cut -d ":" -f 2)" # Check if project is part of the list and get revision if defined build_proj="" sha1="" for p in $PROJECTS ; do IFS='@' read -r -a v <<< "$p" if [ "$proj" = "${v[0]}" ] ; then build_proj="ok" [ ${#v[@]} -eq 2 ] && sha1="${v[1]}" break fi done [ -z "$build_proj" ] && continue # Only clone if the branch exists if has_remote_branch "$REPOSITORY/$proj.git" "$BRANCH"; then git clone -b "$BRANCH" "$REPOSITORY/$proj.git" \ --reference "$REFDIR/$proj" \ "$SRCDIR/$dir" |& tee "$LOGBASE-clone.$LOGEXT" # Make sure we are on the right revision if [ ! -z "$sha1" ] ; then git -C "$SRCDIR/$dir" checkout --detach "$sha1" elif [ ! -z "$REVISION" ]; then # Move down to the right revision update_git "$SRCDIR/$dir" "$REVISION" else # Get the latest revision LLVM_REVS="$LLVM_REVS $(find_svn_rev "$SRCDIR/$dir")" fi # Error if we did specify projects list and branches, but not there else echo "ERROR: Project ($proj) specified in the list, but branch ($BRANCH) not found" CLONE_STATUS=1 break fi done # Get the latest revision from all repos if [ -z "$REVISION" ]; then REVISION="r$(find_last_rev "$LLVM_REVS")" fi } 9>"$REFDIR.lock" # Make sure we unlock the flock before returning if [ "$CLONE_STATUS" -ne 0 ]; then exit $CLONE_STATUS fi # CMake OPTIONS="-DLLVM_BUILD_TESTS=True " OPTIONS+="-DCMAKE_BUILD_TYPE='$BUILDTYPE' " OPTIONS+="-DLLVM_ENABLE_ASSERTIONS='$ASSERTS' " OPTIONS+="-DLLVM_PARALLEL_COMPILE_JOBS='$BUILDJOBS' " OPTIONS+="-DLLVM_PARALLEL_LINK_JOBS='$LINKJOBS' " OPTIONS+="-DCMAKE_INSTALL_PREFIX='$INSTDIR' " OPTIONS+="$CMAKEFLAGS " mkdir -p "$BUILDDIR" cd "$BUILDDIR" && bash -c "cmake -G Ninja '$SRCDIR' $OPTIONS -DLLVM_LIT_ARGS='-sv -j$CPUS'" |& tee "$LOGBASE-cmake.$LOGEXT" # Build + check cd "$BUILDDIR" && ninja "-j$CPUS" |& tee "$LOGBASE-build.$LOGEXT" cd "$BUILDDIR" && ninja "-j$CPUS" check-all |& tee "$LOGBASE-check.$LOGEXT" # Install mkdir -p "$INSTDIR" cd "$BUILDDIR" && ninja "-j$CPUS" install |& tee "$LOGBASE-install.$LOGEXT" # Pack & Ship PKGTAG="$BUILDID-$REVISION" TARGET=$("$BUILDDIR/bin/llc" -version | awk '/target:/ { print $3 }') PKGNAME="$(push_binary_name "$PKGTAG" "$TARGET")" PKGDIR="$WORKSPACE/$PKGNAME" PUSHTYPE="binaries" PUSHDIR="$(push_binary_dir "$PUSHTYPE")" cd "$WORKSPACE" && [ ! -d "$PKGDIR" ] && mv "$INSTDIR" "$PKGDIR" cd "$WORKSPACE" && tar cJf "$PKGNAME.$PUSHSUFFIX" "$PKGNAME" ssh $PUSHSERVER mkdir -p $PUSHDIR cd "$WORKSPACE" && scp "$PKGNAME.$PUSHSUFFIX" "$(push_scp_url "$PUSHTYPE")" # Dump URL echo "TOOLCHAIN URL: $(push_wget_url "$PKGTAG" "$TARGET" "$PUSHTYPE")" # Create toolchain file (wget URL) if [ "$TOOLCHAIN_FILE" != "" ]; then echo "toolchain=$(push_wget_url "$PKGTAG" "$TARGET" "$PUSHTYPE")" >> "$TOOLCHAIN_FILE" echo "revision=$REVISION" >> "$TOOLCHAIN_FILE" fi