path: root/tcwg-llvm-build.sh

#!/usr/bin/env bash

# This script is an experiment on how to build LLVM in Jenkins.
# The steps here are based on the scripts currently in:
# https://git.linaro.org/toolchain/llvm/linaro-scripts.git

set -exu
set -o pipefail

# Syntax
SYN_WORKSPACE="--workspace=/path/to/workspace"
SYN_REVISION="--revision=rNNNNN (SVN revision, def. tip)"
SYN_BUILDTYPE="--buildtype={Release|Debug} (def. Release)"
SYN_ASSERTS="--asserts={True|False} (def. True)"
SYN_BUILDJOBS="--buildjobs=N (def. CPUS)"
SYN_LINKJOBS="--linkjobs=N (def. RAM+1)"
SYN_TOOLCHAIN="--toolchain=http://url/for/tarball"
SYN_BUILDID="--buildid=<text-to-add-to-tarball>"
SYNTAX="$0 $SYN_WORKSPACE $SYN_REVISION $SYN_ASSERTS $SYN_LINKJOBS $SYN_BUILDJOBS $SYN_TOOLCHAIN $SYN_BUILDID"

# Environment Variables and default values
BASEDIR=$(dirname "$(readlink -f $0)")
WORKSPACE=$(pwd)
REVISION=
BUILDTYPE="Release"
ASSERTS="True"
CPUS=$(nproc --all)
SRCDIR=""
BUILDDIR=""
INSTDIR=""
BUILDJOBS=$CPUS
LINKJOBS=$(free -g | awk '/Mem/ {print $2}')
TOOLCHAIN=""
BUILDID=""
CC=""
CXX=""
# We don't want more link jobs than CPUs, even if there's plenty of RAM
if [ "$LINKJOBS" -ge "$CPUS" ]; then
  LINKJOBS=$CPUS
# We may use between 500MB and 1GB per link job, though
else
  LINKJOBS=$((LINKJOBS+1))
fi

# Command line parsing
while [ "$#" -gt 0 ]; do
    ARG=$(echo "$1" | cut -d "=" -f 1)
    VAL=$(echo "$1" | cut -d "=" -f 2)
    case "$ARG" in
      --workspace)
        if [ -d "$VAL" ]; then
          WORKSPACE="$VAL"
        else
          echo "ERROR: $1"
          echo "Syntax: $SYN_WORKSPACE"
          exit 1
        fi
        shift ;;
      --revision)
        if [ ! -z "$VAL" ]; then
          if echo "$VAL" | grep -q "r[0-9]\+"; then
            REVISION="$VAL"
          else
            echo "ERROR: $1"
            echo "Syntax: $SYN_REVISION"
            exit 1
          fi
        fi
        shift ;;
      --buildid)
        if [ "$VAL" != "" ]; then
          BUILDID="$VAL"
        else
          echo "ERROR: $1"
          echo "Syntax: $SYN_BUILDID"
          exit 1
        fi
        shift ;;
      --buildtype)
        if [ "$VAL" = "Debug" ]; then
          BUILDTYPE="$VAL"
        elif [ "$VAL" != "Release" ]; then
          echo "ERROR: $1"
          echo "Syntax: $SYN_BUILDTYPE"
          exit 1
        fi
        shift ;;
      --asserts)
        if [ "$VAL" = "False" ]; then
          ASSERTS="$VAL"
        elif [ "$VAL" != "True" ]; then
          echo "ERROR: $1"
          echo "Syntax: $SYN_ASSERTS"
          exit 1
        fi
        shift ;;
      --buildjobs)
        if [ "$VAL" -gt 0 ]; then
          BUILDJOBS="$VAL"
        else
          echo "ERROR: Build jobs '$VAL' not valid"
          echo "Syntax: $SYN_BUILDJOBS"
          exit 1
        fi
        shift ;;
      --linkjobs)
        if [ "$VAL" -gt 0 ]; then
          LINKJOBS="$VAL"
        else
          echo "ERROR: Link jobs '$VAL' not valid"
          echo "Syntax: $SYN_LINKJOBS"
          exit 1
        fi
        shift ;;
      --toolchain)
        if [ "$VAL" != "" ]; then
          TOOLCHAIN="$VAL"
          file=$(basename "$VAL")
          dir=$(echo "$file" | sed 's/\.tar.*//g')
          # Download the toolchain
          if [ ! -d "$WORKSPACE/$dir" ]; then
            echo "No dir '$WORKSPACE/$dir'..."
            if [ ! -f "$WORKSPACE/$file" ]; then
              echo "No file '$WORKSPACE/$file'..."
              echo "Downloading from '$VAL'"
              wget -q "$VAL" -O "$WORKSPACE/$file"
            fi
            echo "Unpacking '$WORKSPACE/$file'"
            tar xf "$WORKSPACE/$file" -C "$WORKSPACE"
          fi
          # Clang or GCC?
          if [ -f "$WORKSPACE/$dir/bin/clang" ]; then
            export CC="$WORKSPACE/$dir/bin/clang"
            export CXX="$WORKSPACE/$dir/bin/clang++"
          elif [ -f "$WORKSPACE/$dir/bin/gcc" ]; then
            export CC="$WORKSPACE/$dir/bin/gcc"
            export CXX="$WORKSPACE/$dir/bin/g++"
          else
            echo "ERROR: Toolchain '$VAL' has no known compiler in $dir/bin"
            echo "Syntax: $SYN_TOOLCHAIN"
            exit 1
          fi
        else
          echo "WARNING: No toolchain specified, using the system one"
        fi
        shift ;;
      *)
        echo "ERROR: Invalid argument '$1'"
        echo "Syntax: $SYNTAX"
        exit 1
        ;;
    esac
done

# Validate options
if [ "$WORKSPACE" = "" ] || [ ! -d "$WORKSPACE" ]; then
  echo "ERROR: Missing workspace"
  echo "$SYNTAX"
  exit 1
fi
if [ "$BUILDTYPE" = "" ] || [ "$ASSERTS" = "" ]; then
  echo "ERROR: Missing build type / asserts option"
  echo "$SYNTAX"
  exit 1
fi
if [ "$LINKJOBS" = "" ] || [ "$BUILDJOBS" = "" ]; then
  echo "ERROR: Missing number of build / link jobs"
  echo "$SYNTAX"
  exit 1
fi

# Dirs
SRCDIR="$WORKSPACE/llvm"
BUILDDIR="$WORKSPACE/build"
INSTDIR="$WORKSPACE/install"
REFDIR="$HOME/llvm-reference"

# Dump
echo "REVISION = $REVISION"
echo "BUILDTYPE = $BUILDTYPE"
echo "ASSERTS = $ASSERTS"
echo "SRCDIR = $SRCDIR"
echo "BUILDDIR = $BUILDDIR"
echo "INSTDIR = $INSTDIR"
echo "REFDIR = $REFDIR"
echo "BUILDJOBS = $BUILDJOBS"
echo "LINKJOBS = $LINKJOBS"
echo "TOOLCHAIN = $TOOLCHAIN"
echo "CC = $CC"
echo "CXX = $CXX"

# Logs
LOGBASE="$WORKSPACE/tcwg-llvm"
LOGEXT="txt"

# Checkout
{
  flock -s 9
  git clone --depth 1 http://llvm.org/git/llvm.git \
            --reference "$REFDIR/llvm" \
            "$SRCDIR" |& tee "$LOGBASE-clone.$LOGEXT"
  git clone --depth 1 http://llvm.org/git/clang.git \
            --reference "$REFDIR/clang" \
            "$SRCDIR/tools/clang" |& tee -a "$LOGBASE-clone.$LOGEXT"
} 9>$REFDIR.lock

update_git() {
  base=$1
  rev=$2

  # Find the closest git hash for the SVN revision
  pushd "$base"
  hash=$("$BASEDIR/svn-git-hash.pl" "$rev" | awk '{print $2}')

  # We don't need to create a branch with the commit, as the clones are new
  # Should be ok to stay in a detached head state for this purpose
  git checkout "$hash"
  popd
}

# Move down to the right revision
if [ ! -z "$REVISION" ]; then
  update_git "$SRCDIR" "$REVISION"
  update_git "$SRCDIR/tools/clang" "$REVISION"
fi

# CMake
OPTIONS="-DLLVM_BUILD_TESTS=True "
OPTIONS+="-DCMAKE_BUILD_TYPE='$BUILDTYPE' "
OPTIONS+="-DLLVM_ENABLE_ASSERTIONS='$ASSERTS' "
OPTIONS+="-DLLVM_PARALLEL_COMPILE_JOBS='$BUILDJOBS' "
OPTIONS+="-DLLVM_PARALLEL_LINK_JOBS='$LINKJOBS' "
OPTIONS+="-DCMAKE_INSTALL_PREFIX='$INSTDIR' "
mkdir -p "$BUILDDIR"
cd "$BUILDDIR" && cmake -G Ninja "$SRCDIR" $OPTIONS -DLLVM_LIT_ARGS="-sv -j$CPUS" |& tee "$LOGBASE-cmake.$LOGEXT"

# Build + check
cd "$BUILDDIR" && ninja "-j$CPUS" |& tee "$LOGBASE-build.$LOGEXT"
cd "$BUILDDIR" && ninja "-j$CPUS" check-all |& tee "$LOGBASE-check.$LOGEXT"

# Install
mkdir -p "$INSTDIR"
cd "$BUILDDIR" && ninja "-j$CPUS" install |& tee "$LOGBASE-install.$LOGEXT"

# Pack & Ship
VERSION=$("$BUILDDIR/bin/clang" --version | awk '/clang version/ { print $3 }')
TARGET=$("$BUILDDIR/bin/clang" --version | awk '/Target:/ { print $2 }')
PKGNAME=clang+llvm-$VERSION-ci$BUILDID-$TARGET
PKGDIR="$WORKSPACE/$PKGNAME"
PUSHDIR=public_html/builds/binaries/$PKGNAME
FILESERVER=dev-01.tcwglab

cd "$WORKSPACE" && [ ! -d "$PKGDIR" ] && mv "$INSTDIR" "$PKGDIR"
cd "$WORKSPACE" && tar cf "$PKGNAME.tar.xz" "$PKGNAME"
ssh $FILESERVER mkdir -p $PUSHDIR
cd "$WORKSPACE" && scp "$PKGNAME.tar.xz" $FILESERVER:"$PUSHDIR"