#!/usr/bin/env bash # This script is a common repository for TCWG LLVM Jenkins logic, mainly: # - Creating tarball names, URLs # - Downloading toolchains, recognising compilers # - Generic board attributes (cores, memory, etc) # FIXME: Most of this logic should go to the LLVM scripts # Verify and download the toolchain, if any download_toolchain() { local WORKSPACE="$1" local TOOLCHAIN="$2" if [ -z "$TOOLCHAIN" ]; then echo "WARNING: No toolchain specified, using the system one" return 0 fi if [ "${CC+set}" = "set" ] || [ "${CXX+set}" = "set" ]; then echo "ERROR: Duplicating CC/CXX setup" exit 1 fi # Basename/dir local FILE local DIR FILE=$(basename "$TOOLCHAIN") DIR=${FILE//\.tar.*//} # Download the toolchain if [ ! -d "$WORKSPACE/$DIR" ]; then echo "No dir '$WORKSPACE/$DIR'..." if [ ! -f "$WORKSPACE/$FILE" ]; then echo "No file '$WORKSPACE/$FILE'..." echo "Downloading from '$TOOLCHAIN'" wget --progress=dot:giga "$TOOLCHAIN" -O "$WORKSPACE/$FILE" fi if [ ! -s "$WORKSPACE/$FILE" ]; then echo "ERROR: Failed to retrieve toolchain '$TOOLCHAIN'" rm -f "$WORKSPACE/$FILE" exit 1 fi echo "Unpacking '$WORKSPACE/$FILE'" tar xf "$WORKSPACE/$FILE" -C "$WORKSPACE" fi if [ ! -d "$WORKSPACE/$DIR/bin" ]; then echo "ERROR: Toolchain '$TOOLCHAIN' has no 'bin' directory" exit 1 fi # Clang or GCC? if [ -f "$WORKSPACE/$DIR/bin/clang" ]; then export CC="$WORKSPACE/$DIR/bin/clang" export CXX="$WORKSPACE/$DIR/bin/clang++" elif [ -f "$WORKSPACE/$DIR/bin/gcc" ]; then export CC="$WORKSPACE/$DIR/bin/gcc" export CXX="$WORKSPACE/$DIR/bin/g++" else echo "ERROR: Toolchain '$TOOLCHAIN' has no known compiler in $DIR/bin" exit 1 fi } # Set the compiler based on the name (clang/gcc) set_compiler () { local VAL="$1" if [ "${CC+set}" = "set" ] || [ "${CXX+set}" = "set" ]; then echo "ERROR: Duplicating CC/CXX setup" exit 1 fi if [ "$VAL" = "clang" ]; then export CC="clang" export CXX="clang++" elif [ "$VAL" = "gcc" ]; then export CC="gcc" export CXX="g++" else echo "ERROR: Unknown compiler '$VAL'" exit 1 fi } # Find the closest git hash for the SVN revision update_git() { local BASE=$1 local REV=$2 local BASEDIR BASEDIR=$(dirname "$(readlink -f "$0")") # We don't need to create a branch with the commit, as the clones are new # Should be ok to stay in a detached head state for this purpose hash=$("$BASEDIR/svn-git-hash.pl" "$BASE" "$REV" | awk '{print $2}') git -C "$BASE" checkout --detach "$hash" } # Find the SVN revision of a git-svn repository # The first commit might not be upstream in a branch # Assuming there won't be more than 10 non-upstream # If this becomes common, we'll have to move this logic to a perl/python script find_svn_rev() { local BASE=$1 rev="$(git -C "$BASE" log -n 10 | grep git-svn-id | perl -pe "s/.*@(\d+)\s.*/\$1/")" echo "$rev" } # Find the largest revision of a list find_last_rev() { local LAST=0 for rev in "$@"; do [ "$rev" -gt "$LAST" ] && LAST=$rev done echo "$LAST" } # Git remote has branch has_remote_branch() { local REPO="$1" local BRANCH="$2" REF="$(git ls-remote --heads "$REPO" "$BRANCH")" if [ -z "$REF" ]; then return 1 else return 0 fi } # Find element in array is_in_list() { local ITEM="$1" shift for item in "$@"; do if [ "$ITEM" = "$item" ]; then return 0 fi done return 1 } # This list is used in every place we need info about the projects LLVM_PROJECTS="llvm: clang:tools/clang lld:tools/lld compiler-rt:projects/compiler-rt libunwind:projects/libunwind libcxx:projects/libcxx libcxxabi:projects/libcxxabi openmp:projects/openmp" # List all projects with a branch list_branch_projects() { local BRANCH="$1" local BRANCHED_PROJECTS=() local REP="https://git-us.linaro.org/toolchain/llvm" for proj_ref in $LLVM_PROJECTS; do proj="$(echo "$proj_ref" | cut -d ":" -f 1)" if has_remote_branch "$REP/$proj.git" "$BRANCH"; then BRANCHED_PROJECTS+=("$proj") fi done echo "${BRANCHED_PROJECTS[@]}" } # Environment Variables and default values CPUS=$(nproc --all) # We may use between 500MB and 1GB per link job LINKJOBS=$(($(free -g | awk '/Mem/ {print $2}')+1)) if [ "$LINKJOBS" -gt "$CPUS" ]; then LINKJOBS=$CPUS fi # Temporary location for toolchains produced (pushed) PUSHPREFIX=clang+llvm PUSHSUFFIX=tar.xz PUSHSERVER=bkp-01.tcwglab PUSHUSER=tcwg-buildslave push_binary_name() { local TAG="$1" # ex: ci12, 4.0.0-rc1, r300293 local TARGET="$2" # ex: aarch64-linux-gnu echo "$PUSHPREFIX-$TAG-$TARGET" } push_binary_dir() { local TYPE="$1" # ex: releases/binaries echo "builds/$TYPE/llvm" } push_scp_url() { local TYPE="$1" # ex: releases/binaries local PUSHDIR PUSHDIR="$(push_binary_dir "$TYPE")" echo "$PUSHSERVER:$PUSHDIR" } push_wget_url() { local TAG="$1" # ex: ci12, 4.0.0-rc1, r300293 local TARGET="$2" # ex: aarch64-linux-gnu local TYPE="$3" # ex: releases/binaries local BINARY BINARY="$(push_binary_name "$TAG" "$TARGET")" local PUSHDIR PUSHDIR="$(push_binary_dir "$TYPE")" echo "http://$PUSHSERVER/~$PUSHUSER/$PUSHDIR/$BINARY.$PUSHSUFFIX" }