summaryrefslogtreecommitdiff
path: root/MakeRelease.job
blob: a60f8a0b0deba29388b5de146b9251d5c3d2c7be (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
#!/bin/bash

set -x

usage() {
    ret=1
    [ x"$1" != x ] && ret=$1
    echo "MakeRelease: [options]"
    echo "     --abedir - PATH to ABE"
    echo "     --workspace - Specify an alternate workspace"
    echo "     --target XXX - Specify the target triplet to build"
    echo "     --release_name XXX - Specify the release name"
    echo "     --toolchainconfig XXX - [Optional] Explicitly define which toolchain"
    echo "                             to build, e.g., 'default', 'gcc5', 'gcc6'.  "
    echo "                             If not specified, 'default' is the default"
    echo "     --fileserver XXX - Specify the fileserver for tarballs"
    echo "     --manifest XXX - Optionally use a manifest as input"
    echo "     --glibc XXX - Specify Glibc version to build"
    echo "     --gcc XXX - Specify GCC version to build"
    echo "     --binutils XXX - Specify Binutils version to build"
    echo "     --binariesdir XXX - Specify where to upload the generated binaries"
    echo "     --logsdir XXX - Specify where to upload the logs"
    echo "     --canadian - Perform a Canadian-cross build too"
    echo "     --buildnumber XXX - Specify build number"
    echo "     --help"
    exit $ret
}

read_var()
{
   local artifact_list=$1
   local var=$2
   grep "^${var}=" "${artifact_list}" | cut -d = -f 2-
}

# Jenkins may run only using environment variables, a user needs to always
# supply some command line arguments.
if test $# -eq 0 -a x"${JENKINS_SERVER_COOKIE}" = x; then
    usage
fi

# Set defaults
# shellcheck disable=SC2154
if test x"${debug}" = x"true"; then
    export CONFIG_SHELL="/bin/bash -x"
else
    export CONFIG_SHELL="/bin/bash"
fi
buildnumber=0
target=""
fileserver=""
extra=
toolchain_config=""
user_workspace="${WORKSPACE-/home/${USER-buildslave}/workspace}"
abe_dir=
binariesdir=
logsdir=
canadian=false
gcc_src=
binutils_src=
glibc_src=
manifest_src=
toolchain_config=
artifacts_top="releases"
manifest_validation=true

getopt -o h -l target:,release_name:,fileserver:,workspace:,toolchainconfig:,manifest:,glibc:,gcc:,binutils:,help,abedir:,binariesdir:,logsdir:,canadian,buildnumber:,artifacts_top:,manifest_validation: -Q
while test $# -gt 0; do
    case $1 in
	--abedir) abe_dir=$2 ; shift ;;
	--workspace) user_workspace=$2 ; shift ;;
	--target) target=$2 ; shift ;;
	--release_name) release_name=$2 ; shift ;;
	--toolchainconfig) toolchain_config=$2 ; shift ;;
        --fileserver) fileserver=$2 ; shift ;;
        --manifest) manifest_src=$2 ; shift ;;
        --glibc) glibc_src=$2 ; shift ;;
        --gcc) gcc_src=$2 ; shift ;;
        --binutils) binutils_src=$2 ; shift ;;
        --binariesdir) binariesdir=$2 ; shift ;;
        --logsdir) logsdir=$2 ; shift ;;
	--canadian) canadian=true ;;
        --buildnumber) buildnumber=$2 ; shift ;;
	--artifacts_top) artifacts_top=$2; shift ;;
	--manifest_validation) manifest_validation=$2; shift ;;
        -h|--help) usage 0 ;;
	--) break ;;
        *) usage ;;
    esac
    shift
done

if test x"${abe_dir}" = x; then
    echo "Error: --abedir missing"
    usage
fi

if test x"${fileserver}" = x; then
    rsh="sh -c"
    rsync_flags=""
else
    # Hardcode port 22 for the time being.  We use this parameter to
    # upload release tarballs to dev-01, but our ssh default config
    # points to the host container where the directory with binary
    # tarballs is not mapped.
    rsh="ssh -p 22 $fileserver"
    rsync_flags="ssh -p 22"
fi

user_snapshots="${user_workspace}/snapshots"

if test -e ${user_workspace}; then
    cat << EOF > ${user_workspace}/BUILD-INFO.txt
Format-Version: 0.5

Files-Pattern: *
License-Type: open
EOF
fi

# Create a build directory
if test ! -d ${user_workspace}/_build; then
    mkdir -p ${user_workspace}/_build
else
    rm -fr ${user_workspace}/_build/*
fi

# By default, always update all sources
update=""

if test x"${toolchain_config}" != x"" \
	-a x"${toolchain_config}" != x"default"; then
    extra="${extra} --extraconfigdir $abe_dir/config/${toolchain_config}"
fi

if [ x"$target" = x"aarch64-linux-gnu_ilp32" ]; then
    extra="$extra --extraconfigdir $abe_dir/config/ilp32-dev"
fi

# Use the newly created build directory
cd ${user_workspace}/_build || exit

if ! test -e host.conf; then
  $CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} \
      --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref
fi

if test x"${release_name}" != x; then
    release="${release_name}"
else
    echo "MakeRelease.job must be invoked with a --release_name <release> string."
    exit 1
fi

manifest=${manifest_src:+--manifest ${manifest_src}}
glibc=${glibc_src:+glibc=${glibc_src}}
binutils=${binutils_src:+binutils=${binutils_src}}
gcc=${gcc_src:+gcc=${gcc_src}}
srcs="${gcc} ${binutils} ${glibc} ${manifest}"
logfile=${user_workspace}/MakeRelease-${buildnumber}-$(uname -m).log

# Set ABE's --target setting.  No setting means native.
# If manifest file is set, then ABE must use its setting.
target_opt=
if [ x"${target}" != x"native" ] && [ x"${target}" != x ] && \
   [ x"$manifest" = x"" ]; then
    target_opt="--target ${target}"
fi

# Build a binary release tarball
# Remove logfile if present (for some unknown reason)
rm -f ${logfile}
abe_ret=0
manifests=()
tarballs=()
# Canadian cross builds require a Linux hosted cross compiler first
if test x"${canadian}" = x"true"; then
    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} $target_opt --build all ${extra} --tarbin >> ${logfile}
    abe_ret=$?
    host="--host i686-w64-mingw32"
    manifests+=( "$(read_var ${user_workspace}/artifacts1.txt manifest)" )
else
    host=""
fi

# If we're not building the mingw32 compiler 'abe_ret' will be zero and the
# following conditional will build the linux cross-compiler.  Otherwise it'll
# build the mingw32 compiler only if the previous cross-compiler build was
# successful.
if test ${abe_ret} -eq 0; then
    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} $target_opt ${host} --build all ${extra} >> ${logfile}
    abe_ret=$?
    manifests+=( "$(read_var ${user_workspace}/artifacts2.txt manifest)" )
fi


if test x"${canadian}" = x"true"; then
    # take the toolchain, sysroot and runtime tarballs from the linux build
    mapfile -t -O "${#tarballs[@]}" tarballs < <( \
      read_var ${user_workspace}/artifacts1.txt '\(toolchain\|sysroot\|runtime\)\(_asc\)\?' )
    # and just the toolchain from the mingw build
    mapfile -t -O "${#tarballs[@]}" tarballs < <( \
      read_var ${user_workspace}/artifacts2.txt 'toolchain\(_asc\)\?' )
else
    mapfile -t -O "${#tarballs[@]}" tarballs < <( \
      read_var ${user_workspace}/artifacts2.txt '\(toolchain\|sysroot\|runtime\)\(_asc\)\?' )
fi

# MakeRelease.job doesn't require an input gcc_src parameter (it'll invoke ABE
# with the default) so we can't rely on it for the gcc version.  Parse the
# generated manifest file name instead, e.g.,
#    gcc-linaro-6.1.1-2016.08-rc1-linux-manifest.txt
#    gcc-linaro-6.1.1-2016.08-rc1-win32-manifest.txt
manifest_match="*-manifest.txt"

# Make sure we have at least one manifest file found.
if test -z "${manifests[0]}"; then
    echo "Couldn't find a manifest file for the recent build."
    exit 1
fi

# Major.Minor.Point
gcc_full_version=`basename ${manifests[0]} | awk -F '-' '{ print $3 }'`

# Before GCC 5 the GCC release name was Major.Minor.Point.  With GCC 5
# and later the GCC release name is Major.Minor.
gcc_major=`echo $gcc_full_version | awk -F '.' '{ print $1 }'`
gcc_minor=`echo $gcc_full_version | awk -F '.' '{ print $2 }'`
gcc_release=${gcc_major}.${gcc_minor}
if test ${gcc_major} -lt 5; then
    gcc_point=`echo $gcc_full_version | awk -F '.' '{ print $3 }'`
    gcc_release=${gcc_major}.${gcc_minor}.${gcc_point}
fi

# Strip off everything but the DATE[-SPIN][-RC], complicated by the fact
# that the spin and rc are both optional and not mutually exclusive.
series_date=${manifests[0]##*-${gcc_full_version}-}
series_date=${series_date%${manifest_match}}

# We aren't sure if we're getting the linux or win32 manifest so strip both.
series_date=${series_date%-linux}
series_date=${series_date%-win32}

# Fileserver location of binary tarballs and build logfiles
binaries="$artifacts_top/binaries/${gcc_release}-${series_date}-${buildnumber}/${target}"
logs="$artifacts_top/logs/${gcc_release}-${series_date}-${buildnumber}/${target}"

if [ "x${binariesdir}" != "x" ] && \
    [ "x${binariesdir}" != "x${binaries}" ]; then
    echo "WARNING: Provided binaries dirname does not match with what was computed"
    echo "Provided: ${binariesdir}"
    echo "Computed: ${binaries}"
    binaries=${binariesdir}
fi

if [ "x${logsdir}" != "x" ] && \
     [ "x${logsdir}" != "x${logs}" ]; then
    echo "WARNING: Provided logs dirname does not match with what was computed"
    echo "Provided: ${logsdir}"
    echo "Computed: ${logs}"
    logs=${logsdir}
fi

# Copy the build log to the fileserver
if test -e ${logfile}; then
    echo "Compressing log file..."
    xz ${logfile}
    $rsh "if test ! -d ${logs}; then mkdir -p ${logs}; fi"
    echo "Copying compressed log file ${logfile}.xz to ${fileserver:+$fileserver:}${logs}..."
    rsync -a ${rsync_flags:+-e "$rsync_flags"} ${logfile}.xz ${fileserver:+$fileserver:}${logs}/
fi

# force a failure if abe has build problems.
if test ${abe_ret} -ne 0; then
  exit 1
fi

# Copy the binaries to the fileserver
$rsh "if test ! -d ${binaries}; then mkdir -p ${binaries}; fi"

# There could be a linux and/or a win32 manifest.
for manifest in "${manifests[@]}"
do
    rsync -a ${rsync_flags:+-e "$rsync_flags"} ${manifest} ${fileserver:+$fileserver:}${binaries}/
done

rsync -a ${rsync_flags:+-e "$rsync_flags"} "${tarballs[@]}" ${fileserver:+$fileserver:}${binaries}/

if $manifest_validation; then
    echo "Starting manifest validation."
    git clone --depth 1 git://git.linaro.org/toolchain/abe-tests.git
    for manifest in "${manifests[@]}"; do
	./abe-tests/validate-manifest.pl "${manifest}"
	if test $? -ne 0; then
	    echo "Manifest validation failed."
	    exit 1
	else
	    echo "Manifest validation passed."
	fi
    done
fi

exit 0