summaryrefslogtreecommitdiff
path: root/tcwg-llvm-common.sh
blob: f410efe615ca32e7ea7f1bcc88d2115a1c8bd18d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#!/usr/bin/env bash

# This script is a common repository for TCWG LLVM Jenkins logic, mainly:
#  - Creating tarball names, URLs
#  - Downloading toolchains, recognising compilers
#  - Generic board attributes (cores, memory, etc)
# FIXME: Most of this logic should go to the LLVM scripts

# Verify and download the toolchain, if any
download_toolchain() {
  local WORKSPACE="$1"
  local TOOLCHAIN="$2"
  if [ -z "$TOOLCHAIN" ]; then
    echo "WARNING: No toolchain specified, using the system one"
    return 0
  fi
  if [ "${CC+set}" = "set" ] || [ "${CXX+set}" = "set" ]; then
    echo "ERROR: Duplicating CC/CXX setup"
    exit 1
  fi
  # Basename/dir
  local FILE
  local DIR
  FILE=$(basename "$TOOLCHAIN")
  DIR=${FILE//\.tar.*//}
  # Download the toolchain
  if [ ! -d "$WORKSPACE/$DIR" ]; then
    echo "No dir '$WORKSPACE/$DIR'..."
    if [ ! -f "$WORKSPACE/$FILE" ]; then
      echo "No file '$WORKSPACE/$FILE'..."
      echo "Downloading from '$TOOLCHAIN'"
      wget --progress=dot:giga "$TOOLCHAIN" -O "$WORKSPACE/$FILE"
    fi
    if [ ! -s "$WORKSPACE/$FILE" ]; then
      echo "ERROR: Failed to retrieve toolchain '$TOOLCHAIN'"
      rm -f "$WORKSPACE/$FILE"
      exit 1
    fi
    echo "Unpacking '$WORKSPACE/$FILE'"
    tar xf "$WORKSPACE/$FILE" -C "$WORKSPACE"
  fi
  if [ ! -d "$WORKSPACE/$DIR/bin" ]; then
    echo "ERROR: Toolchain '$TOOLCHAIN' has no 'bin' directory"
    exit 1
  fi
  # Clang or GCC?
  if [ -f "$WORKSPACE/$DIR/bin/clang" ]; then
    export CC="$WORKSPACE/$DIR/bin/clang"
    export CXX="$WORKSPACE/$DIR/bin/clang++"
  elif [ -f "$WORKSPACE/$DIR/bin/gcc" ]; then
    export CC="$WORKSPACE/$DIR/bin/gcc"
    export CXX="$WORKSPACE/$DIR/bin/g++"
  else
    echo "ERROR: Toolchain '$TOOLCHAIN' has no known compiler in $DIR/bin"
    exit 1
  fi
}

# Set the compiler based on the name (clang/gcc)
set_compiler () {
  local VAL="$1"
  if [ "${CC+set}" = "set" ] || [ "${CXX+set}" = "set" ]; then
    echo "ERROR: Duplicating CC/CXX setup"
    exit 1
  fi
  if [ "$VAL" = "clang" ]; then
    export CC="clang"
    export CXX="clang++"
  elif [ "$VAL" = "gcc" ]; then
    export CC="gcc"
    export CXX="g++"
  else
    echo "ERROR: Unknown compiler '$VAL'"
    exit 1
  fi
}

# Find the closest git hash for the SVN revision
update_git() {
  local BASE=$1
  local REV=$2
  local BASEDIR
  BASEDIR=$(dirname "$(readlink -f "$0")")

  # We don't need to create a branch with the commit, as the clones are new
  # Should be ok to stay in a detached head state for this purpose
  hash=$("$BASEDIR/svn-git-hash.pl" "$BASE" "$REV" | awk '{print $2}')
  git -C "$BASE" checkout --detach "$hash"
}

# Find the SVN revision of a git-svn repository
# The first commit might not be upstream in a branch
# Assuming there won't be more than 10 non-upstream
# If this becomes common, we'll have to move this logic to a perl/python script
find_svn_rev() {
  local BASE=$1
  rev="$(git -C "$BASE" log -n 10 | grep git-svn-id | perl -pe "s/.*@(\d+)\s.*/\$1/")"
  echo "$rev"
}

# Find the largest revision of a list
find_last_rev() {
  local LAST=0
  for rev in "$@"; do
    [ "$rev" -gt "$LAST" ] && LAST=$rev
  done
  echo "$LAST"
}

# Git remote has branch
has_remote_branch() {
  local REPO="$1"
  local BRANCH="$2"
  REF="$(git ls-remote --heads "$REPO" "$BRANCH")"
  if [ -z "$REF" ]; then
    return 1
  else
    return 0
  fi
}

# Find element in array
is_in_list() {
  local ITEM="$1"
  shift
  for item in "$@"; do
    if [ "$ITEM" = "$item" ]; then
      return 0
    fi
  done
  return 1
}

# This list is used in every place we need info about the projects
LLVM_PROJECTS="llvm: clang:tools/clang lld:tools/lld
               compiler-rt:projects/compiler-rt libunwind:projects/libunwind
               libcxx:projects/libcxx libcxxabi:projects/libcxxabi
               openmp:projects/openmp"
# List all projects with a branch
list_branch_projects() {
  local BRANCH="$1"
  local BRANCHED_PROJECTS=()
  local REP="https://git-us.linaro.org/toolchain/llvm"
  for proj_ref in $LLVM_PROJECTS; do
    proj="$(echo "$proj_ref" | cut -d ":" -f 1)"
    if has_remote_branch "$REP/$proj.git" "$BRANCH"; then
      BRANCHED_PROJECTS+=("$proj")
    fi
  done
  echo "${BRANCHED_PROJECTS[@]}"
}

# Environment Variables and default values
CPUS=$(nproc --all)
# We may use between 500MB and 1GB per link job
LINKJOBS=$(($(free -g | awk '/Mem/ {print $2}')+1))
if [ "$LINKJOBS" -gt "$CPUS" ]; then
  LINKJOBS=$CPUS
fi
# Temporary location for toolchains produced (pushed)
PUSHPREFIX=clang+llvm
PUSHSUFFIX=tar.xz
PUSHSERVER=bkp-01.tcwglab
PUSHUSER=tcwg-buildslave

push_binary_name() {
  local TAG="$1"     # ex: ci12, 4.0.0-rc1, r300293
  local TARGET="$2"  # ex: aarch64-linux-gnu
  echo "$PUSHPREFIX-$TAG-$TARGET"
}

push_binary_dir() {
  local TYPE="$1"    # ex: releases/binaries
  echo "builds/$TYPE/llvm"
}

push_scp_url() {
  local TYPE="$1"    # ex: releases/binaries
  local PUSHDIR
  PUSHDIR="$(push_binary_dir "$TYPE")"
  echo "$PUSHSERVER:$PUSHDIR"
}

push_wget_url() {
  local TAG="$1"     # ex: ci12, 4.0.0-rc1, r300293
  local TARGET="$2"  # ex: aarch64-linux-gnu
  local TYPE="$3"    # ex: releases/binaries
  local BINARY
  BINARY="$(push_binary_name "$TAG" "$TARGET")"
  local PUSHDIR
  PUSHDIR="$(push_binary_dir "$TYPE")"
  echo "http://$PUSHSERVER/~$PUSHUSER/$PUSHDIR/$BINARY.$PUSHSUFFIX"
}