tcwg-llvm-*: Only clone the repos that have the specified branch

This implements the simplest way to do a pre-commit test as described in
TCWG-1124: Any tree that needs building will push to the appropriate
branch all the projects that they need, regardless of changes.

This can only work if the underlying process guarantees that:

1. No user will ever create a repo with that name, even by accident,
for example, by Forcing an exclusive namespace like
jenkins-test/username/branch.

2. The branches are deleted at the end, regardless of success or failure.
To re-start a build on the same branch, just push it again and re-run the
job. This may not be possible for Jenkins to do (as it doesn't have
write permissions to all git repos we could possibly try to build from),
so this will need some coordination with the rest of the LLVM scripts.

This change also re-factors how to checkout the llvm repositories, so
that we can control on a branch level and collect only the relevant SVN
revisions from the branches we have actually checked out.

Change-Id: I2f77ac49f23ce79f2057f1cc90e1ccd63e2a45f4

2 files changed, 42 insertions, 19 deletions

diff --git a/tcwg-llvm-build.sh b/tcwg-llvm-build.sh
index 66215dc3..8b4e05a2 100755
--- a/tcwg-llvm-build.sh
+++ b/tcwg-llvm-build.sh
@@ -37,7 +37,7 @@ BUILDJOBS=$CPUS
 TOOLCHAIN=""
 TOOLCHAIN_FILE=""
 BUILDID=""
-REPOSITORY="https://llvm.org/git/llvm.git"
+REPOSITORY="https://git-us.linaro.org/toolchain/llvm"
 BRANCH="master"
 
 # Command line parsing
@@ -198,26 +198,34 @@ LOGBASE="$WORKSPACE/tcwg-llvm"
 LOGEXT="txt"
 
 # Checkout
+LLVM_PROJECTS="llvm: clang:tools/clang lld:tools/lld
+               compiler-rt:projects/compiler-rt libunwind:projects/libunwind
+               libcxx:projects/libcxx libcxxabi:projects/libcxxabi"
+LLVM_REVS=""
 {
   flock -s 9
-  git clone -b "$BRANCH" "$REPOSITORY/llvm.git" \
-            --reference "$REFDIR/llvm" \
-            "$SRCDIR" |& tee "$LOGBASE-clone.$LOGEXT"
-  git clone -b "$BRANCH" "$REPOSITORY/clang.git" \
-            --reference "$REFDIR/clang" \
-            "$SRCDIR/tools/clang" |& tee -a "$LOGBASE-clone.$LOGEXT"
-} 9>"$REFDIR.lock"
-
-if [ ! -z "$REVISION" ]; then
-  # Move down to the right revision
-  update_git "$SRCDIR" "$REVISION"
-  update_git "$SRCDIR/tools/clang" "$REVISION"
-else
+  for proj_ref in $LLVM_PROJECTS; do
+    proj="$(echo "$proj_ref" | cut -d ":" -f 1)"
+    dir="$(echo "$proj_ref" | cut -d ":" -f 2)"
+    if has_remote_branch "$REPOSITORY/$proj.git" "$BRANCH"; then
+      git clone -b "$BRANCH" "$REPOSITORY/$proj.git" \
+                --reference "$REFDIR/$proj" \
+                "$SRCDIR/$dir" |& tee "$LOGBASE-clone.$LOGEXT"
+      # Make sure we are on the right revision
+      if [ ! -z "$REVISION" ]; then
+        # Move down to the right revision
+        update_git "$SRCDIR/$dir" "$REVISION"
+      else
+        # Get the latest revision
+        LLVM_REVS="$LLVM_REVS $(find_svn_rev "$SRCDIR/$dir")"
+      fi
+    fi
+  done
   # Get the latest revision from all repos
-  LLVM_REV="$(find_svn_rev "$SRCDIR")"
-  CLANG_REV="$(find_svn_rev "$SRCDIR/tools/clang")"
-  REVISION="r$(find_last_rev "$LLVM_REV" "$CLANG_REV")"
-fi
+  if [ -z "$REVISION" ]; then
+    REVISION="r$(find_last_rev $LLVM_REVS)"
+  fi
+} 9>"$REFDIR.lock"
 
 # CMake
 OPTIONS="-DLLVM_BUILD_TESTS=True "
diff --git a/tcwg-llvm-common.sh b/tcwg-llvm-common.sh
index 7eaec6db..180faa88 100755
--- a/tcwg-llvm-common.sh
+++ b/tcwg-llvm-common.sh
@@ -88,9 +88,12 @@ update_git() {
 }
 
 # Find the SVN revision of a git-svn repository
+# The first commit might not be upstream in a branch
+# Assuming there won't be more than 10 non-upstream
+# If this becomes common, we'll have to move this logic to a perl/python script
 find_svn_rev() {
   local BASE=$1
-  rev="$(git -C "$BASE" log -n 1 | grep git-svn-id | perl -pe "s/.*@(\d+)\s.*/\$1/")"
+  rev="$(git -C "$BASE" log -n 10 | grep git-svn-id | perl -pe "s/.*@(\d+)\s.*/\$1/")"
   echo "$rev"
 }
 
@@ -103,6 +106,18 @@ find_last_rev() {
   echo "$LAST"
 }
 
+# Git remote has branch
+has_remote_branch() {
+  local REPO="$1"
+  local BRANCH="$2"
+  REF="$(git ls-remote --heads "$REPO" "$BRANCH")"
+  if [ -z "$REF" ]; then
+    return 1
+  else
+    return 0
+  fi
+}
+
 # Environment Variables and default values
 CPUS=$(nproc --all)
 # We may use between 500MB and 1GB per link job