#!/usr/bin/env bash # This script is an experiment on how to build LLVM in Jenkins. # The steps here are based on the scripts currently in: # https://git.linaro.org/toolchain/llvm/linaro-scripts.git set -e set -o pipefail # Syntax SYN_BUILD="--build={ninja|make} (def. make)" SYN_BUILDTYPE="--buildtype={Release|Debug} (def. Release)" SYN_ASSERTS="--asserts={True|False} (def. True)" SYN_SRCDIR="--srcdir=path/to/source" SYN_BUILDDIR="--builddir=path/to/build" SYN_INSTDIR="--instdir=path/to/install" SYN_BUILDJOBS="--buildjobs=N (def. CPUS)" SYN_LINKJOBS="--linkjobs=N (def. RAM+1)" SYN_TOOLCHAIN="--toolchain=http://url/for/tarball" SYNTAX="$0 $SYN_BUILD $SYN_ASSERTS $SYN_RELEASETYPE $SYN_SRCDIR $SYN_BUILDDIR $SYN_INSTDIR $SYN_LINKJOBS $SYN_BUILDJOBS $STN_TOOLCHAIN" # Environment Variables and default values GEN="Unix Makefiles" BUILD="make" BUILDTYPE="Release" ASSERTS="True" CPUS=$(nproc --all) SRCDIR="" BUILDDIR="" INSTDIR="" BUILDJOBS=$CPUS LINKJOBS=$(free -g | awk '/Mem/ {print $2}') TOOLCHAIN="" # We don't want more link jobs than CPUs, even if there's plenty of RAM if [ "$LINKJOBS" -ge "$CPUS" ]; then LINKJOBS=$CPUS # We may use between 500MB and 1GB per link job, though else LINKJOBS=$((LINKJOBS+1)) fi # Command line parsing while [ "$#" -gt 0 ]; do ARG=$(echo "$1" | cut -d "=" -f 1) VAL=$(echo "$1" | cut -d "=" -f 2) case "$ARG" in --build) if [ "$VAL" = "ninja" ]; then GEN="Ninja" BUILD="ninja" elif [ "$VAL" != "make" ]; then echo "ERROR: $1" echo "Syntax: $SYN_BUILD" exit 1 fi shift ;; --buildtype) if [ "$VAL" = "Debug" ]; then BUILDTYPE="$VAL" elif [ "$VAL" != "Release" ]; then echo "ERROR: $1" echo "Syntax: $SYN_BUILDTYPE" exit 1 fi shift ;; --asserts) if [ "$VAL" = "False" ]; then ASSERTS="$VAL" elif [ "$VAL" != "True" ]; then echo "ERROR: $1" echo "Syntax: $SYN_ASSERTS" exit 1 fi shift ;; --srcdir) if [ -d "$VAL" ]; then SRCDIR="$VAL" else echo "ERROR: Source dir '$VAL' doesn't exist" echo "Syntax: $SYN_SRCDIR" exit 1 fi shift ;; --builddir) if [ ! -d "$VAL" ] && ! mkdir -p "$VAL"; then echo "ERROR: Build dir '$VAL' doesn't exist and can't be created" echo "Syntax: $SYN_BUILDDIR" exit 1 fi BUILDDIR="$VAL" shift ;; --instdir) if [ ! -d "$VAL" ] && ! mkdir -p "$VAL"; then echo "ERROR: Install dir '$VAL' doesn't exist and can't be created" echo "Syntax: $SYN_INSTDIR" exit 1 fi INSTDIR="$VAL" shift ;; --buildjobs) if [ "$VAL" -gt 0 ]; then BUILDJOBS="$VAL" else echo "ERROR: Build jobs '$VAL' not valid" echo "Syntax: $SYN_BUILDJOBS" exit 1 fi shift ;; --linkjobs) if [ "$VAL" -gt 0 ]; then LINKJOBS="$VAL" else echo "ERROR: Link jobs '$VAL' not valid" echo "Syntax: $SYN_LINKJOBS" exit 1 fi shift ;; --toolchain) if [ "$VAL" != "" ]; then file=$(basename "$VAL") dir=$(echo "$file" | sed 's/\.tar.*//g') # Download the toolchain if [ ! -d "$WORKSPACE/$dir" ]; then echo "No dir '$WORKSPACE/$dir'..." if [ ! -f "$WORKSPACE/$file" ]; then echo "No file '$WORKSPACE/$file'..." echo "Downloading from '$VAL'" wget -q "$VAL" -O "$WORKSPACE/$file" fi echo "Unpacking '$WORKSPACE/$file'" tar xf "$WORKSPACE/$file" -C "$WORKSPACE" fi # Clang or GCC? if [ -f "$WORKSPACE/$dir/bin/clang" ]; then export CC="$WORKSPACE/$dir/bin/clang" export CXX="$WORKSPACE/$dir/bin/clang++" elif [ -f "$WORKSPACE/$dir/bin/gcc" ]; then export CC="$WORKSPACE/$dir/bin/gcc" export CXX="$WORKSPACE/$dir/bin/g++" else echo "ERROR: Toolchain '$VAL' has no known compiler in $dir/bin" echo "Syntax: $SYN_TOOLCHAIN" exit 1 fi else echo "ERROR: Toolchain '$VAL' not valid" echo "Syntax: $SYN_LINKJOBS" exit 1 fi shift ;; *) echo "ERROR: Invalid argument '$1'" echo "Syntax: $SYNTAX" exit 1 ;; esac done # Validate options if [ "$GEN" = "" ] || [ "$BUILD" = "" ]; then echo "ERROR: Missing Generator / Builder" echo "$SYNTAX" exit 1 fi if [ "$BUILDTYPE" = "" ] || [ "$ASSERTS" = "" ]; then echo "ERROR: Missing build type / asserts option" echo "$SYNTAX" exit 1 fi if [ "$BUILDDIR" = "" ] && [ "$SRCDIR" != "" ]; then BUILDDIR="$SRCDIR/../build" if [ ! -d "BUILDDIR" ] && ! mkdir -p "$BUILDDIR"; then echo "ERROR: Forced build dir '$BUILDDIR' can't be created" echo "Syntax: $SYN_BUILDDIR" exit 1 fi fi if [ "$INSTDIR" = "" ] && [ "$BUILDDIR" != "" ]; then INSTDIR="$BUILDDIR/../install" if [ ! -d "INSTDIR" ] && ! mkdir -p "$INSTDIR"; then echo "ERROR: Forced install dir '$INSTDIR' can't be created" echo "Syntax: $SYN_INSTDIR" exit 1 fi fi if [ "$SRCDIR" = "" ] || [ "$BUILDDIR" = "" ] || [ "$INSTDIR" = "" ]; then echo "ERROR: Missing source / build / install directories" echo "$SYNTAX" exit 1 fi if [ "$LINKJOBS" = "" ] || [ "$BUILDJOBS" = "" ]; then echo "ERROR: Missing number of build / link jobs" echo "$SYNTAX" exit 1 fi # Dump echo "GEN = $GEN" echo "BUILD = $BUILD" echo "BUILDTYPE = $BUILDTYPE" echo "ASSERTS = $ASSERTS" echo "SRCDIR = $SRCDIR" echo "BUILDDIR = $BUILDDIR" echo "INSTDIR = $INSTDIR" echo "BUILDJOBS = $BUILDJOBS" echo "LINKJOBS = $LINKJOBS" echo "TOOLCHAIN = $TOOLCHAIN" echo "CC = $CC" echo "CXX = $CXX" # Logs LOGBASE="$SRCDIR/../tcwg-llvm" LOGEXT="txt" # Checkout git clone http://llvm.org/git/llvm.git "$SRCDIR" |& tee "$LOGBASE-clone.$LOGEXT" git clone http://llvm.org/git/clang.git "$SRCDIR/tools/clang" |& tee -a "$LOGBASE-clone.$LOGEXT" # CMake OPTIONS="-DLLVM_BUILD_TESTS=True " OPTIONS+="-DCMAKE_BUILD_TYPE='$BUILDTYPE' " OPTIONS+="-DLLVM_ENABLE_ASSERTIONS='$ASSERTS' " OPTIONS+="-DLLVM_PARALLEL_COMPILE_JOBS='$BUILDJOBS' " OPTIONS+="-DLLVM_PARALLEL_LINK_JOBS='$LINKJOBS' " cd "$BUILDDIR" && cmake -G "$GEN" "$SRCDIR" $OPTIONS -DLLVM_LIT_ARGS="-sv -j$CPUS" |& tee "$LOGBASE-cmake.$LOGEXT" # Build + check cd "$BUILDDIR" && $BUILD "-j$CPUS" |& tee "$LOGBASE-build.$LOGEXT" cd "$BUILDDIR" && $BUILD "-j$CPUS" check-all |& tee "$LOGBASE-check.$LOGEXT" # Install DESTDIR="" if [ "$BUILD" = "make" ]; then DESTDIR="DESTDIR=$INSTDIR" fi cd "$BUILDDIR" && $BUILD "-j$CPUS" "$DESTDIR" install |& tee "$LOGBASE-install.$LOGEXT" # Pack & Ship VERSION=$("$BUILDDIR/bin/clang" --version | awk '/clang version/ { print $3 }') TARGET=$("$BUILDDIR/bin/clang" --version | awk '/Target:/ { print $2 }') TARBALL=clang+llvm-$VERSION-ci$BUILD_ID-$TARGET PUSHDIR=public_html/builds/binaries/$TARBALL FILESERVER=dev-01.tcwglab cd "$INSTDIR" && ln -s usr/local "$TARBALL" cd "$INSTDIR" && tar hcf "$TARBALL.tar.xz" "$TARBALL" ssh $FILESERVER mkdir -p $PUSHDIR cd "$INSTDIR" && scp "$TARBALL.tar.xz" $FILESERVER:"$PUSHDIR"