aboutsummaryrefslogtreecommitdiff
path: root/control/setup-control-node
blob: 4f8b922d3a00bf541f017d2f7d6951365cb01eae (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
#!/bin/bash

###############################################################################
# Copyright (c) 2011 Linaro
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
###############################################################################

set -xe
. sandbox-create.conf
MYPATH=$PWD/`dirname $0`
HOSTNAME=$1
[ -z "$HOSTNAME" ] && HOSTNAME=`hostname`
REPO_MIRROR="--repo-url=http://android.git.linaro.org/git-ro/tools/repo.git"

#
# +++ BASE +++
#

sudo apt-get update -qq
sudo apt-get install -qq -y git-core gnupg bzr make

#
# +++ JENKINS +++
#
sudo apt-get install -qq -y python-lxml default-jre daemon
wget -nv -O - http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add -
#echo 'deb http://pkg.jenkins-ci.org/debian binary/' | sudo tee /etc/apt/sources.list.d/jenkins-ci.list > /dev/null
#sudo apt-get update -qq
#sudo apt-get install -qq -y jenkins
wget -nv $JENKINS_DEB
# This might fail due to missing dependencies
sudo dpkg -i $(basename $JENKINS_DEB) || true
# Make apt-get install missing dependencies
sudo apt-get install -qq -f

#sudo sed -i 's%HTTP_PORT=8080%HTTP_PORT=9090%' /etc/default/jenkins
sudo sed -i 's%JENKINS_ARGS="%&--prefix=/jenkins %' /etc/default/jenkins
# Install predefined sparse Jenkins config
sudo cp -rf $MYPATH/jenkins-config/* ~jenkins
# Set web host name
sudo python - ~jenkins/hudson.tasks.Mailer.xml "https://$HOSTNAME/jenkins/" <<EOF
import sys
from lxml import etree
tree = etree.fromstring(open(sys.argv[1]).read())
tag = tree.xpath("/hudson.tasks.Mailer_-DescriptorImpl/hudsonUrl")[0]
tag.text = sys.argv[2]
f = open(sys.argv[1], "w")
f.write(etree.tostring(tree, xml_declaration=True, encoding='UTF-8'))
f.close()
EOF

# Install EC2 plugins
sudo mkdir -p ~jenkins/plugins/
savedir=$PWD
cd ~jenkins/plugins/
for url in $PLUGINS; do
    sudo wget -nv --no-check-certificate $url || exit 1
done
cd $savedir

# Fix up perms
sudo chmod a+xr ~jenkins
sudo chown -R jenkins ~jenkins
sudo service jenkins restart
ls -l ~jenkins/plugins/

# +++ APACHE +++

# enable mod proxy
sudo apt-get install -qq -y apache2 libapache2-mod-wsgi
sudo a2enmod proxy
sudo a2enmod proxy_http
sudo a2enmod headers
sudo a2enmod rewrite
sudo a2enmod expires
sudo a2enmod ssl

# make self-signed certificate
cat <<EOF > /tmp/ssleay.cnf
RANDFILE                = /dev/urandom
[ req ]
default_bits            = 1024
default_keyfile         = privkey.pem
distinguished_name      = req_distinguished_name
prompt                  = no
policy			= policy_anything
[ req_distinguished_name ]
commonName                      = android-build.linaro.org
EOF
CERT_PATH=/etc/ssl/private/android-build.linaro.org.crt
sudo openssl req -config /tmp/ssleay.cnf -new -x509 -days 3650 -nodes -out $CERT_PATH -keyout $CERT_PATH
sudo chmod 0600 $CERT_PATH
rm /tmp/ssleay.cnf

cat <<EOF | sudo tee /etc/apache2/conf.d/extra-port > /dev/null
Listen 127.0.0.1:600
EOF

cat <<\EOF | sudo tee /etc/apache2/sites-available/android-build.linaro.org > /dev/null
<VirtualHost _default_:443>
    ServerAdmin webmaster@localhost
    ServerName android-build.linaro.org
    <Proxy *>
        Order deny,allow
        Allow from all
    </Proxy>

    RewriteEngine on
    RewriteRule ^/builds/~([a-z][-a-z0-9]+)/([-A-Za-z0-9_.]+)/([0-9]+)/output(.*) http://127.0.0.1:600/$1_$2/builds/$3/archive$4 [L,P]
    RewriteRule ^/builds/~([a-z][-a-z0-9]+)/([-A-Za-z0-9_.]+)/lastSuccessful/output(.*) http://127.0.0.1:600/$1_$2/lastSuccessful/archive$3 [L,P]
    RewriteRule ^/mockup(.*) $1 [R=301]

    ProxyPass /jenkins http://localhost:8080/jenkins
    ProxyPassReverse /jenkins http://localhost:8080/jenkins

    Alias /static /home/build-system-frontend/frontend/static
    Alias /3.3.0/build /home/build-system-frontend/yui/build

    WSGIScriptAlias /combo /home/build-system-frontend/lazr-js/combo.wsgi
    WSGIScriptAlias / /home/build-system-frontend/frontend/linaro-abs-frontend.wsgi

    <LocationMatch "/(3.3.0/build|combo)">
        SetOutputFilter DEFLATE
        FileETag none
        ExpiresActive on
        ExpiresDefault "access plus 10 years"
        Header append Cache-Control "public"
    </LocationMatch>

    SSLEngine on
    SSLCertificateFile    /etc/ssl/private/android-build.linaro.org.crt

    BrowserMatch "MSIE [2-6]" \
            nokeepalive ssl-unclean-shutdown \
            downgrade-1.0 force-response-1.0
    BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
</VirtualHost>

<VirtualHost 127.0.0.1:600>
    ServerAdmin webmaster@localhost
    ServerName android-build.linaro.org
    DocumentRoot /var/lib/jenkins/jobs
</VirtualHost>

<VirtualHost _default_:80>
    ServerAdmin webmaster@localhost
    ServerName android-build.linaro.org
    Alias /download/ /var/lib/jenkins/jobs/
    # Provide shortcut URLs to access artifacts
    AliasMatch ^/builds/~([^/]+)/([^/]+)/(lastStable|lastSuccessful)(.*) /var/lib/jenkins/jobs/$1_$2/$3/archive/build/out$4
    AliasMatch ^/builds/~([^/]+)/([^/]+)/([^/]+)(.*) /var/lib/jenkins/jobs/$1_$2/builds/$3/archive/build/out$4
    <Directory "/var/lib/jenkins/jobs/">
        Options Indexes MultiViews FollowSymLinks
        AllowOverride None
    </Directory>
    Alias /seed/ /mnt2/seed/
    <Directory "/mnt2/seed/">
        Options Indexes MultiViews FollowSymLinks
        AllowOverride None
#        Order deny,allow
#        Deny from all
#        Allow from 127.0.0.0/8 ::1/128 10.0.0.0/8
    </Directory>

#    RewriteLog /var/log/apache2/mod_rewrite_log
#    RewriteLogLevel 3

    RewriteEngine on
    RewriteCond %{REQUEST_URI} !^/(download|seed|builds)
    RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [redirect=301,last]
    # Allow to access everything in /builds/*
    RewriteCond %{REQUEST_URI} ^/builds/
    RewriteRule .* - [last]
    # Allow to access XMLs in build artifact archives
    RewriteCond %{REQUEST_URI} .+/archive/.+\.xml$
    RewriteRule .* - [last]
    # But disallow access any other XMLs (e.g. configs)
    RewriteCond %{REQUEST_URI} .+\.xml$
    RewriteRule .* - [forbidden]
</VirtualHost>
EOF

sudo a2ensite android-build.linaro.org
sudo apache2ctl graceful

#
# +++ android mirror +++
#
sudo adduser --system git-mirror
wget 'http://android.git.linaro.org/gitweb?p=tools/repo.git;a=blob_plain;f=repo;hb=refs/heads/stable' -O repo
sudo mv repo /usr/local/bin/repo
sudo chmod a+x /usr/local/bin/repo

sudo mkdir -p /mnt/mirror
sudo chown git-mirror /mnt/mirror/

#sudo -H -u git-mirror mkdir -p /mnt/mirror/android.git.kernel.org
#cd /mnt/mirror/android.git.kernel.org
#sudo -H -u git-mirror /usr/local/bin/repo init -u git://android.git.kernel.org/platform/manifest.git --mirror
#sudo -H -u git-mirror /usr/local/bin/repo sync -j 8

sudo rm -rf /mnt/mirror/pristine
sudo -H -u git-mirror mkdir /mnt/mirror/pristine
cd /mnt/mirror/pristine
sudo -H -u git-mirror /usr/local/bin/repo init $REPO_MIRROR -u git://android.git.linaro.org/platform/manifest.git --mirror


#
# +++ git: daemon +++
#
sudo apt-get install -qq -y git-daemon-run
cat <<EOF | sudo tee /etc/sv/git-daemon/run > /dev/null
#!/bin/sh
exec 2>&1
echo 'git-daemon starting.'
exec chpst -ugitdaemon \
  "$(git --exec-path)"/git-daemon --verbose --base-path=/mnt/mirror --export-all
EOF

# Ignore warning: git-daemon: unable to open supervise/ok: file does not exist
sudo bash -c "sv restart git-daemon; true"

#
# +++ git-mirror service +++
#
cd
if [ -f ~git-mirror/service.pid ]; then
    sudo kill `sudo cat ~git-mirror/service.pid` || true
fi
sudo rm -rf linaro-android-mirror
bzr branch lp:linaro-android-mirror
cd linaro-android-mirror
sudo -H -u git-mirror twistd -y mirror-service.tac --umask=0022 --pid ~git-mirror/service.pid --logfile ~git-mirror/service.log
# Should set up crontab to launch this on @restart or something

#
# +++ gitweb +++
#
# sudo apt-get install -y gitweb

#
# +++ android build service frontend +++
#

sudo apt-get install -qq -y python-virtualenv python-lxml python-openid tidy
sudo adduser --system build-system-frontend
sudo mkdir -p /var/lib/linaro-abs-frontend/
sudo chown www-data /var/lib/linaro-abs-frontend/
sudo -H -u www-data bash -xes <<\EOF
umask 0077
openssl rand 30 | base64 -w0 > /var/lib/linaro-abs-frontend/jenkins-password
EOF

# Update frontend's passwd in Jenkins
sudo $MYPATH/set-jenkins-user-passwd \
    ~jenkins/users/frontend/config.xml /var/lib/linaro-abs-frontend/jenkins-password

sudo -H -u build-system-frontend bash -xes <<\EOF
cd
rm -rf frontend
git clone http://git.linaro.org/git/infrastructure/linaro-android-frontend.git frontend
cd frontend
make
KEY="$(openssl rand 30 | base64 -w0)"
cat > ../settings_prod.py <<EOF2
from settings import *

MEDIA_URL = '/static/'
LOGIN_URL = '/openid/login/'
LOGOUT_URL = '/logout'

DATABASES['default']['NAME'] = '/var/lib/linaro-abs-frontend/session.db'

FRONTEND_JENKINS_USER = 'frontend'
FRONTEND_JENKINS_PASSWORD = open('/var/lib/linaro-abs-frontend/jenkins-password').read().strip()

SECRET_KEY = '$KEY'

EOF2
ln -s ../settings_prod.py .
EOF

cd ~build-system-frontend/frontend
yes no | sudo -u www-data ./bin/manage syncdb --settings settings_prod
sudo -u www-data ./bin/manage loaddata --settings settings_prod group-fixture.json

sudo apt-get install -qq -y unzip python-cssutils
sudo -H -u build-system-frontend sh -xes "$@" <<\EOF
cd
wget -c -nv http://yui.zenfs.com/releases/yui3/yui_3.3.0.zip
unzip -o -q yui_3.3.0.zip
rm -rf lazr-js
bzr branch lp:~mwhudson/lazr-js/combo-mod_wsgi-config lazr-js
cd lazr-js
mkdir -p build/3.3.0
ln -s ~/yui/build build/3.3.0/build
mkdir -p build/gallery
GALLERY_BASE=http://yui.yahooapis.com/combo?gallery-2010.12.16-18-24/build/
fetch_gallery () {
  mkdir -p build/gallery/${1}
  curl $GALLERY_BASE/${1}/${1}-min.js > build/gallery/${1}/${1}-min.js
}
fetch_gallery gallery-overlay-extras
fetch_gallery gallery-outside-events
fetch_gallery gallery-base64
EOF