From aee561ce46291ce59297d6b2e3c6f34e0d0db15a Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 10 Dec 2013 19:07:20 +0200 Subject: Most of utils migrated to https://git.linaro.org/infrastructure/linaro-jenkins-tools.git --- utils/build-stats/build-times.py | 19 - utils/build-stats/buildlog-datamine.py | 112 --- utils/build-stats/jenkins-total-builds | 4 - utils/build-stats/jenkins-total-duration | 5 - utils/cmdline-client/README | 62 -- utils/cmdline-client/android-build-client | 199 ----- utils/compare-builds/compare-builds.py | 30 - utils/diskspace/expire-builds.groovy | 19 - utils/diskspace/watch-space | 2 - utils/mangle-jobs/README | 92 --- utils/mangle-jobs/add-copy-to-slave.mangle | 25 - .../mangle-jobs/add-email-ext-notification.mangle | 135 --- .../mangle-jobs/add-snapshots-l-o-publisher.mangle | 68 -- utils/mangle-jobs/artifacts-add.mangle | 19 - utils/mangle-jobs/artifacts-remove.mangle | 13 - utils/mangle-jobs/artifacts-set.mangle | 15 - utils/mangle-jobs/build-expiration-set.mangle | 17 - utils/mangle-jobs/build-steps-set.mangle | 19 - utils/mangle-jobs/build-timeout-add.mangle | 22 - utils/mangle-jobs/build-timeout-set.mangle | 15 - utils/mangle-jobs/builders.xml | 17 - utils/mangle-jobs/ci-tools-checkout.mangle | 9 - utils/mangle-jobs/copytoslave-remove.mangle | 10 - utils/mangle-jobs/cron-set-no-weekend.mangle | 12 - utils/mangle-jobs/description-prepend.mangle | 28 - .../mangle-jobs/email-ext-notification-set.mangle | 20 - utils/mangle-jobs/fingerprinter-add.mangle | 25 - utils/mangle-jobs/header.html | 12 - utils/mangle-jobs/job-shell-commands-set.mangle | 11 - utils/mangle-jobs/job-shell-commands-set2.mangle | 11 - utils/mangle-jobs/logparser-add.mangle | 20 - utils/mangle-jobs/mangle-jobs | 224 ----- utils/mangle-jobs/mangle_helper.py | 48 -- utils/mangle-jobs/null.mangle | 4 - utils/mangle-jobs/project-based-sec-add.mangle | 17 - utils/mangle-jobs/push-artifacts-set.mangle | 26 - utils/mangle-jobs/reset-cron.mangle | 8 - utils/mangle-jobs/restricted-relabel.mangle | 11 - utils/mangle-jobs/shell-status-set.mangle | 11 - utils/mangle-jobs/test.mangle | 7 - utils/migrate-artifacts/builds-on-snapshots | 911 --------------------- utils/migrate-artifacts/config.py | 18 - utils/migrate-artifacts/sync-builds.py | 113 --- utils/new-publish/README | 111 --- utils/new-publish/clean-uploads | 6 - utils/new-publish/propagate.py | 87 -- utils/new-publish/publib.py | 126 --- utils/new-publish/publish | 71 -- utils/new-publish/setup.sh | 124 --- utils/new-publish/test/data/dir1/dir1_file | 0 utils/new-publish/test/data/dir2/dir2_file | 0 .../test/data/dir2/subdir1/dir2_subdir1_file | 0 utils/new-publish/test/data/top_file | 1 - utils/new-publish/test/test_publish.py | 73 -- utils/query-jobs/README | 25 - utils/query-jobs/query-jobs | 67 -- utils/slave-logs/archive-slave-logs | 19 - utils/slave-logs/jenkins-clean-slave-logs | 4 - utils/slave-logs/merge-logs.sh | 18 - 59 files changed, 3197 deletions(-) delete mode 100755 utils/build-stats/build-times.py delete mode 100755 utils/build-stats/buildlog-datamine.py delete mode 100755 utils/build-stats/jenkins-total-builds delete mode 100755 utils/build-stats/jenkins-total-duration delete mode 100644 utils/cmdline-client/README delete mode 100755 utils/cmdline-client/android-build-client delete mode 100644 utils/compare-builds/compare-builds.py delete mode 100644 utils/diskspace/expire-builds.groovy delete mode 100755 utils/diskspace/watch-space delete mode 100644 utils/mangle-jobs/README delete mode 100644 utils/mangle-jobs/add-copy-to-slave.mangle delete mode 100644 utils/mangle-jobs/add-email-ext-notification.mangle delete mode 100644 utils/mangle-jobs/add-snapshots-l-o-publisher.mangle delete mode 100644 utils/mangle-jobs/artifacts-add.mangle delete mode 100644 utils/mangle-jobs/artifacts-remove.mangle delete mode 100644 utils/mangle-jobs/artifacts-set.mangle delete mode 100644 utils/mangle-jobs/build-expiration-set.mangle delete mode 100644 utils/mangle-jobs/build-steps-set.mangle delete mode 100644 utils/mangle-jobs/build-timeout-add.mangle delete mode 100644 utils/mangle-jobs/build-timeout-set.mangle delete mode 100644 utils/mangle-jobs/builders.xml delete mode 100644 utils/mangle-jobs/ci-tools-checkout.mangle delete mode 100644 utils/mangle-jobs/copytoslave-remove.mangle delete mode 100644 utils/mangle-jobs/cron-set-no-weekend.mangle delete mode 100644 utils/mangle-jobs/description-prepend.mangle delete mode 100644 utils/mangle-jobs/email-ext-notification-set.mangle delete mode 100644 utils/mangle-jobs/fingerprinter-add.mangle delete mode 100644 utils/mangle-jobs/header.html delete mode 100644 utils/mangle-jobs/job-shell-commands-set.mangle delete mode 100644 utils/mangle-jobs/job-shell-commands-set2.mangle delete mode 100644 utils/mangle-jobs/logparser-add.mangle delete mode 100755 utils/mangle-jobs/mangle-jobs delete mode 100644 utils/mangle-jobs/mangle_helper.py delete mode 100644 utils/mangle-jobs/null.mangle delete mode 100644 utils/mangle-jobs/project-based-sec-add.mangle delete mode 100644 utils/mangle-jobs/push-artifacts-set.mangle delete mode 100644 utils/mangle-jobs/reset-cron.mangle delete mode 100644 utils/mangle-jobs/restricted-relabel.mangle delete mode 100644 utils/mangle-jobs/shell-status-set.mangle delete mode 100644 utils/mangle-jobs/test.mangle delete mode 100644 utils/migrate-artifacts/builds-on-snapshots delete mode 100755 utils/migrate-artifacts/config.py delete mode 100755 utils/migrate-artifacts/sync-builds.py delete mode 100644 utils/new-publish/README delete mode 100755 utils/new-publish/clean-uploads delete mode 100755 utils/new-publish/propagate.py delete mode 100755 utils/new-publish/publib.py delete mode 100755 utils/new-publish/publish delete mode 100755 utils/new-publish/setup.sh delete mode 100644 utils/new-publish/test/data/dir1/dir1_file delete mode 100644 utils/new-publish/test/data/dir2/dir2_file delete mode 100644 utils/new-publish/test/data/dir2/subdir1/dir2_subdir1_file delete mode 100644 utils/new-publish/test/data/top_file delete mode 100644 utils/new-publish/test/test_publish.py delete mode 100644 utils/query-jobs/README delete mode 100755 utils/query-jobs/query-jobs delete mode 100755 utils/slave-logs/archive-slave-logs delete mode 100755 utils/slave-logs/jenkins-clean-slave-logs delete mode 100755 utils/slave-logs/merge-logs.sh diff --git a/utils/build-stats/build-times.py b/utils/build-stats/build-times.py deleted file mode 100755 index 752eade..0000000 --- a/utils/build-stats/build-times.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -import sys -import urllib2 -import json -import time - - -URL = "https://android-build.linaro.org/jenkins/api/json" - -f = urllib2.urlopen(URL) -server_info = json.load(f) -for job in server_info["jobs"]: - f = urllib2.urlopen(job["url"] + "/api/json") - job_info = json.load(f) - for build in job_info["builds"]: - f = urllib2.urlopen(build["url"] + "/api/json") - build_info = json.load(f) - tstamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(build_info["timestamp"] / 1000)) - print "%s,%s,%s" % (job["name"], build["number"], tstamp) diff --git a/utils/build-stats/buildlog-datamine.py b/utils/build-stats/buildlog-datamine.py deleted file mode 100755 index 3c66f6a..0000000 --- a/utils/build-stats/buildlog-datamine.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import re -import datetime -import optparse - - -JOBS_ROOT = "/var/lib/jenkins/jobs/" - -# TODO: move this into separate "script" file, allow to pass such script -# to execute by the main processor -PARSE = [ -{"re": r"curl.+/seed/", "capture": "seed_url", "state": "SEED_DL"}, -{"if": "SEED_DL", "re": "^real\t(?P.+)", "state": ""}, - -{"re": "^Checking out files", "state": "REPO_CO"}, -{"if": "REPO_CO", "re": "^real\t(?P.+)", "state": "END"}, -] - -def parse_time_time(s): - m = re.match(r"(.+)m(.+)s", s) - return datetime.timedelta(minutes=int(m.group(1)), seconds=float(m.group(2))) - -def PROCESS(captures): - if "seed_dl_time" not in captures or "repo_co_time" not in captures: - return False - captures["seed_dl_time"] = parse_time_time(captures["seed_dl_time"]) - captures["repo_co_time"] = parse_time_time(captures["repo_co_time"]) - captures["total_co_time"] = captures["seed_dl_time"] + captures["repo_co_time"] - -REPORT2 = """\ -Download time: %(seed_dl_time)s -Checkout time: %(repo_co_time)s -Total time : %(total_co_time)s -""" -REPORT = """\ -%(job_name)s, %(build_no)s, %(seed_dl_time)s, %(repo_co_time)s, %(total_co_time)s -""" - -#+ curl --silent --show-error http://android-build.linaro.org/seed/pandaboard.mirror.tar.gz -#+ gzip -d -c -#+ tar x - -#real<-->1m50.108s - -class FinishProcessing: - pass - -def process_file(fname, captures): - f = open(fname) - state = "" - - try: - for l in f: - for rule in PARSE: - if "if" not in rule or rule["if"] == state: - m = rule["re"].search(l) - if m: - captures.update(m.groupdict()) - if "state" in rule: - state = rule["state"] - if state == "END": - raise FinishProcessing - except FinishProcessing: - pass - - if PROCESS(captures) == False: - return None - return REPORT % captures - - -def get_build_no(build_path): - f = open(build_path + "/build.xml") - for l in f: - if "" in l: - f.close() - return re.search(r"[0-9]+", l).group(0) - f.close() - - -def process_one_log(fname, job_name, build_no): - captures = {"job_name": job_name, "build_no": build_no} - report = process_file(fname, captures) - if report: - sys.stdout.write(report) - -optparser = optparse.OptionParser(usage="%prog") -optparser.add_option("--pattern", help="Process only jobs matching regex pattern") -optparser.add_option("--file", help="Process one log") -options, args = optparser.parse_args(sys.argv[1:]) -if len(args) != 0: - optparser.error("Wrong number of arguments") - - -# Precompile regexes -for rule in PARSE: - rule["re"] = re.compile(rule["re"]) - -if options.file: - process_one_log(options.file, "unknown-job", "unknown-build") - sys.exit() - -for path, dirs, files in os.walk(JOBS_ROOT): - if "log" in files: - if "build.xml" not in files: - continue - job_name = path[len(JOBS_ROOT):].split("/", 1)[0] - if options.pattern and not re.search(options.pattern, job_name): - continue - build_no = get_build_no(path) - process_one_log(path + "/log", job_name, build_no) diff --git a/utils/build-stats/jenkins-total-builds b/utils/build-stats/jenkins-total-builds deleted file mode 100755 index 5a1c0d5..0000000 --- a/utils/build-stats/jenkins-total-builds +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# Calculate total number of builds present in archive -find /var/lib/jenkins/jobs -name log | wc -l diff --git a/utils/build-stats/jenkins-total-duration b/utils/build-stats/jenkins-total-duration deleted file mode 100755 index 723843a..0000000 --- a/utils/build-stats/jenkins-total-duration +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -JENKINS_ROOT=/var/lib/jenkins/jobs -find $JENKINS_ROOT -name 'build.xml*' | xargs cat | awk '/.+<\/duration>/ { dur += gensub(".*>([0-9]+).*", "\\1", "g"); } -END { printf("%.2fhrs\n", dur / 1000 / 60 / 60); }' diff --git a/utils/cmdline-client/README b/utils/cmdline-client/README deleted file mode 100644 index 9ec0459..0000000 --- a/utils/cmdline-client/README +++ /dev/null @@ -1,62 +0,0 @@ -Basic android-build.linaro.org command-line client -================================================== - -This tool provides proof-of-concept command-line client for Linaro Android -Build Jenkins master (android-build.linaro.org), as an alternative for -existing Web UI interface. One particular feature the command-line client -has and Web UI lacks is support for creating completely private builds (such -builds are not availabel via Web UI at all). At the same time, this tool is -so far in the proof of concept stage and provides only basic job management -actions (namely, create a job and schedule its build). If you find this tool -useful, please share your comments and suggestion using this bugtracker: - -https://bugs.launchpad.net/linaro-android-infrastructure - -to help its improvement and evolution. - - -Quick start ------------ - -1. You should have access to Jenkins at: - -http://android-build.linaro.org/jenkins/ - -(Generally available for Linaro Android team members). - -2. Download android-build-client tool. As it is written to depend only -on Python standard library, an easy way to get it is to download seperate -file via Launchpad BZR viewer: - -wget "http://bazaar.launchpad.net/~linaro-automation/linaro-android-build-tools/trunk/download/head:/androidbuildclient-20130724114943-yefh8succov70g5i-2/android-build-client" -chmod +x android-build-client - -3. Look up your Jenkins API token by visiting -https://android-build.linaro.org/jenkins/me/configure and clicking -"Show API Token..." button. - -4. Run: - -./android-build-client authorize - -Enter yous Jenkins username and API token. Note that this will cache these -credentials in your home dir, so use this only on your personal well-protected -workstation. The alternative is to use --user switch and input API key -interactively. - -5. To create a new job, prepare an Android job config (see -https://wiki.linaro.org/Platform/Android/LinaroAndroidBuildService for more -info) in a file. Run: - -./android-build-client create - -Note that job name should conform to Android Build Service naming -converntions, see documentation link above for more info. To create a job -private for a particular group, pass --private= switch to create command. - -6. To schedule a build, run: - -./android-build-client build - - -Run ./android-build-client --help to see all options. diff --git a/utils/cmdline-client/android-build-client b/utils/cmdline-client/android-build-client deleted file mode 100755 index b236fc5..0000000 --- a/utils/cmdline-client/android-build-client +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python -############################################################################### -# Copyright (c) 2013 Linaro -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# which accompanies this distribution, and is available at -# http://www.eclipse.org/legal/epl-v10.html -############################################################################### - -import base64 -import json -import os -import sys -import urllib2 -import optparse -import getpass -from xml.dom import minidom - -PRIVATE_ACL = """\ - - hudson.model.Item.Build:%(group)s - hudson.model.Item.Configure:%(group)s - hudson.model.Item.Read:%(group)s - hudson.model.Item.Cancel:%(group)s - hudson.model.Item.Discover:%(group)s - -""" - - -class Jenkins(object): - - def __init__(self, base_url, username, passwd): - self.base = base_url - self.auth_headers = { - 'Authorization': 'Basic %s' % ( - base64.b64encode('%s:%s' % (username, passwd)))} - self.csrf = None - - def jenkins_rest(self, jenkins_path, data=None, extra_headers=None): - """Make an authenticated request to jenkins. - - @param jenkins_path: The path on the Jenkins instance to make the - request to. - @param data: Data to include in the request (if this is not None the - request will be a POST). - @param extra_headers: A dictionary of extra headers that will passed in - addition to Authorization. - @raises urllib2.HTTPError: If the response is not a HTTP 200. - @returns: the body of the response. - """ - headers = self.auth_headers.copy() - if extra_headers: - headers.update(extra_headers) - url = self.base + jenkins_path - print url - req = urllib2.Request( - url, data, headers) - resp = urllib2.urlopen(req) - data = resp.read() - resp.close() - return data - - def get_csrf_headers(self): - if self.csrf is None: - try: - crumb_data = self.jenkins_rest('crumbIssuer/api/json') - data = json.loads(crumb_data) - self.csrf = {data['crumbRequestField']: data['crumb']} - except urllib2.HTTPError: - # Ignore errors in case CSRF protection is not enabled - self.csrf = {} - return self.csrf - - def jenkins_rest_post(self, jenkins_path, data, extra_headers=None): - if extra_headers is None: - extra_headers = {} - extra_headers.update(self.get_csrf_headers()) - return self.jenkins_rest(jenkins_path, data, extra_headers) - - def get_job_config(self, job_name): - return self.jenkins_rest('job/' + job_name + '/config.xml') - - def post_config(self, url, config_xml): - return self.jenkins_rest_post(url, config_xml, - {'Content-Type': 'text/xml'}) - - def set_job_config(self, job_name, config_xml): - return self.post_config('job/' + job_name + '/config.xml', config_xml) - - def create_job(self, job_name, config_xml): - return self.post_config('createItem?name=' + job_name, config_xml) - - def build_job(self, job_name): - self.jenkins_rest_post( - 'job/' + job_name + '/buildWithParameters?delay=0sec', '') - - -def job2user_group(job_group): - if job_group == "linaro-android": - return "linaro-android-builders" - return job_group - - -def error(msg): - print >>sys.stderr, msg - sys.exit(1) - - -def main(): - global options - optparser = optparse.OptionParser(usage="%prog authorize|create|build ...") - optparser.add_option("--url", - default="https://android-build.linaro.org/jenkins", - help="Jenkins base url, default: %default") - optparser.add_option("--user", - help="Jenkins username, default: $USER") - optparser.add_option("--apikey-file", metavar="FILE", - help="File holding Jenkins API key") - optparser.add_option("--cred-file", metavar="FILE", - help="File holding Jenkins username:API key pair") - optparser.add_option("--private", metavar="GROUP", - help="Create private job accessible to GROUP") - - options, args = optparser.parse_args(sys.argv[1:]) - if len(args) < 1: - optparser.error("Wrong number of arguments") - - config_dir = os.path.expanduser("~/.config/android-build-client") - - password = None - if options.cred_file: - options.user, password = open(options.cred_file).read().strip().split(":") - elif options.apikey_file: - password = open(options.passwd_file).read().strip() - elif os.path.exists(config_dir + "/cred"): - options.user, password = open(config_dir + "/cred").read().strip().split(":") - print "INFO: Using cached authorization for: %s" % options.user - elif args[0] != "authorize": - password = getpass.getpass("API Token:") - - if options.url[-1] != '/': - options.url += '/' - - j = Jenkins(options.url, options.user, password) - - if args[0] == "authorize": - if len(args) != 1: - optparser.error("Usage: authorize") - import getpass - user = raw_input("Enter user name: ") - while True: - key = getpass.getpass("API key: ") - if len(key) == 32: - break - print "This does not look like Jenkisn API key, please try again" - try: - os.makedirs(config_dir) - except OSError: - pass - f = open(config_dir + "/cred", "w") - print >>f, "%s:%s" % (user, key) - f.close() - print "Credentials cached for future use" - elif args[0] == "create": - if len(args) != 3: - optparser.error("Usage: create ") - job_group, job_subname = args[1].split("/") - template = j.get_job_config("template_" + job2user_group(job_group)) - dom = minidom.parseString(template) - nodes = dom.getElementsByTagName("hudson.model.StringParameterDefinition") - assert len(nodes) == 1 - n = nodes[0].getElementsByTagName("defaultValue")[0] - build_config = open(args[2]).read() - n.childNodes[0].data = base64.encodestring(build_config) - - if options.private: - n = dom.getElementsByTagName("properties")[0] - c = n.getElementsByTagName("hudson.security.AuthorizationMatrixProperty")[0] - acl_dom = minidom.parseString(PRIVATE_ACL % {"group": options.private}) - n.replaceChild(acl_dom.documentElement, c) - - job_conf = dom.toxml() - jenkins_job = args[1].replace("/", "_") - try: - j.create_job(jenkins_job, job_conf) - except urllib2.HTTPError: - error("Error creating job '%s' (job exists?)" % jenkins_job) - print "Job created successfully: %s/job/%s/" % (options.url, jenkins_job) - elif args[0] == "build": - if len(args) != 2: - optparser.error("Usage: build ") - j.build_job(args[1].replace("/", "_")) - print "Build queued" - else: - optparser.error("Unknown command '%s'" % args[0]) - - -if __name__ == "__main__": - main() diff --git a/utils/compare-builds/compare-builds.py b/utils/compare-builds/compare-builds.py deleted file mode 100644 index 9c6d6d4..0000000 --- a/utils/compare-builds/compare-builds.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys -import os - -job = sys.argv[1] -build1 = sys.argv[2] -build2 = sys.argv[3] - -def download(fname, build): - base, ext = fname.split(".", 1) - stamped_fname = "%s-%s.%s" % (base, build, ext) - if os.path.exists(stamped_fname): - print "Using cached %s" % stamped_fname - return - url_pat = "https://android-build.linaro.org/jenkins/job/linaro-android_staging-origen/%s/artifact/build/out/%s" - url = url_pat % (build, fname) - os.system("wget --no-check-certificate '%s' -O %s" % (url, stamped_fname)) - -def diff(fname, build1, build2): - base, ext = fname.split(".", 1) - cmdline = "diff -u %s-%s.%s %s-%s.%s > %s-%s-%s.diff" % (base, build1, ext, base, build2, ext, base, build1, build2) - os.system(cmdline) - - -def compare_text_files(fname, build1, build2): - download(fname, build1) - download(fname, build2) - diff(fname, build1, build2) - -compare_text_files("source-manifest.xml", build1, build2) -compare_text_files("pinned-manifest.xml", build1, build2) diff --git a/utils/diskspace/expire-builds.groovy b/utils/diskspace/expire-builds.groovy deleted file mode 100644 index 5f7dcd7..0000000 --- a/utils/diskspace/expire-builds.groovy +++ /dev/null @@ -1,19 +0,0 @@ -// Force build expiration for all Jenkins jobs -// We need this script because by default expires -// jobs only when thay are actually built. - -// Run by going to https://android-build.linaro.org/jenkins/script -// Or using: -// java -jar jenkins-cli.jar -s http://localhost:8080/jenkins/ -i groovy expire.groovy -// This script is expected to run via cron - -for (job in hudson.model.Hudson.instance.items) { - // That's how you print debug info: - // println(job); - - // Be extra cautious about release builds - if (job.name ==~ ".+(201.+|-release)") - continue; - - job.logRotate(); -} diff --git a/utils/diskspace/watch-space b/utils/diskspace/watch-space deleted file mode 100755 index afec5ca..0000000 --- a/utils/diskspace/watch-space +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -df | awk '$6=="/mnt2" {if ($4 < 1000000) print "Less than 1Gb left in JENKINS_HOME: " $4 "Kb"}' diff --git a/utils/mangle-jobs/README b/utils/mangle-jobs/README deleted file mode 100644 index 70f01da..0000000 --- a/utils/mangle-jobs/README +++ /dev/null @@ -1,92 +0,0 @@ -Intro ------ -mangle-jobs is a scripting framework to perform mass migration of Jenkins -job definitions. The motivataion for such script is that, the way -linaro-android-cloud-buildd is used, all (or large cluster) of jobs -represent the same type of build process, just parameterized with -specific settings (Android and toolchain versions, etc.) Thus, many -changes needed - for example, exporting additional build artifact, or -enabling some plugin - should be applied to all jobs (or large subset) -at the same time. Having an automated scripting framework for such -migrations saves from doing time-consuming and error-prone manual changes -via Jenkins UI for lot of jobs. The drawback is that migration is performed -on the XML-encoded content, so some development and XML processing -familiarity is required. However, due to XPath and LXML library which -implements its support, migration development is sufficiently easy, and -there's growing collection of existing migrations scripts, to serve -as example and templates for new usecases. - -mangle-jobs script ------------------- -mangle-jobs script allows to run a "mangle" (migration) script for all -jobs present in Jenkins and submit results back to server for permanent -storage. A mangle script operates on an XML representation of job config, -and can apply arbitrary Python algorithm to it (including conditionals -to apply to a subset of jobs). Besides production mode of updating jobs -in a networked Jenkins server, mangle-jobs also offers extensive dry-run -and testing capabilities, suitable for mangle script development and -pre-production testing. - -Mangle script development workflow ----------------------------------- -Basic mangle-jobs workflow is as follows: - -1. Figure out what XML changes must be performed. For example, select -a particular job in Jenkins, make a backup copy of its config, make -needed changes via Jenkins UI, and diff job config against the backup -copy. - -2. Find existing mangle script performing changes close to ones needed. -As noted above, there's a good selection of such scripts, and it makes -little sense to develop a new script from scratch, this is perfect -example when program-by-example technique helps. - -3. Make a copy of template you found and make needed changes. - -4. Take the config backup copy you made earlier and run mangle-jobs on it -in test mode: - -$ ./mangle-jobs --file= - -You will see a textual diff of old and new config, highlighting the changes -your script made. Verify that they are as expected, and repeat from step 3 -if needed. - -5. You are ready to perform en-masse pre-production test now. You should have -Jenkins username with appropriate permissions and its password or API key -(recommended) handy. The API key can found in Jenkins by clicking you username -in the top right corner, then Configure in menu. Password/API key will be -prompted on teh console. If you really need that, you can put the credential -in the file and refer to it with --passwd-file= to avoid -interactive prompts. - -$ ./mangle-jobs --user= - -This will run your script repeatedly for each job store in Jenkins server and -will show aggregated diff output. Review it carefully to watch for anomalities -and corner cases. Repeat from step 3 if needed. - -6. Once you're absolutely sure that the changes performed by your mangle script -are correct, run it in the production update mode: - -$ ./mangle-jobs --user= --really - -Mangle script details ---------------------- -A mangle script is standalone Python module. It should define a function -mangle(), accepting single parameter - elemnt tree, parsed from XML job -config with lxml.etree.fromstring(). The function should perform needed -transformation on the tree (in place). - -To get speciifics of command-line arguments and options, run mangle-jobs -with --help option. - -Known issues ------------- -XML dialects produced by Jenkins and by mangle-jobs tool may not necessarily -match. This means that while they may match with respect to underlying XML -syntax (node structure and content), their textual representation may be -different, leading to spurious differences in test run output. There's -ongoing effort to make output XML rendition of mangle-jobs correspond to -the dialect used by Jenkins, but there still may be spurious change noise -in the output. diff --git a/utils/mangle-jobs/add-copy-to-slave.mangle b/utils/mangle-jobs/add-copy-to-slave.mangle deleted file mode 100644 index de28dde..0000000 --- a/utils/mangle-jobs/add-copy-to-slave.mangle +++ /dev/null @@ -1,25 +0,0 @@ -# Add copy-to-slave plugin setup for build dependencies - -#from lxml.etree import Element -from lxml.etree import fromstring - -new_node = """\ - - **/* - - false - false - false - copyToSlave - -""" - -def mangle(tree): - if tree.xpath("//com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper"): - # Already there - return - tag = tree.xpath('/project/buildWrappers')[0] - node = fromstring(new_node) - tag.append(node) - node.getprevious().tail += " " - node.tail = "\n " diff --git a/utils/mangle-jobs/add-email-ext-notification.mangle b/utils/mangle-jobs/add-email-ext-notification.mangle deleted file mode 100644 index a0bea64..0000000 --- a/utils/mangle-jobs/add-email-ext-notification.mangle +++ /dev/null @@ -1,135 +0,0 @@ -# Add Email notifications - -from lxml.etree import fromstring -from mangle_helper import * - - -builders = 'linaro-android-builder-notifications@linaro.org' -restricted_builders = 'linaro-android-restricted-builder-notifications@linaro.org' - -publisher = """\ - - %s - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - - - $PROJECT_DEFAULT_SUBJECT - $PROJECT_DEFAULT_CONTENT - false - false - false - true - - false - - - - - default - $DEFAULT_SUBJECT - $DEFAULT_CONTENT - - - false - $DEFAULT_RECIPIENTS - -""" - -def mangle(tree): - if tree.xpath("//hudson.plugins.emailext.ExtendedEmailPublisher"): - # Already there - return - tag = tree.xpath('/project/publishers')[0] - cfg = get_build_config(tree) - if cfg.get("BUILD_TYPE", "build-android") == "build-android-restricted": - node = fromstring(publisher % restricted_builders) - else: - node = fromstring(publisher % builders) - tag.append(node) - node.getprevious().tail += " " - node.tail = "\n " diff --git a/utils/mangle-jobs/add-snapshots-l-o-publisher.mangle b/utils/mangle-jobs/add-snapshots-l-o-publisher.mangle deleted file mode 100644 index 21cddb8..0000000 --- a/utils/mangle-jobs/add-snapshots-l-o-publisher.mangle +++ /dev/null @@ -1,68 +0,0 @@ -# Add SSH(SFTP) artifact publishing to snapshots.linaro.org - -#from lxml.etree import Element -from lxml.etree import fromstring - -publisher = """\ - - SSH: - - - - snapshots.linaro.org - false - - - ${JOB_NAME}/${BUILD_NUMBER} - build/out/target/*/*/*.img,build/out/target/*/*/*.tar.bz2,build/out/target/*/*/MD5SUMS,build/out/*.tar.bz2,build/out/*.xml,build/out/*_config,build/out/lava-job-info - - build/out - false - false - false - - 120000 - false - - - false - false - - - snapshots.linaro.org file-move - false - - - - - - - false - false - false - reshuffle-files - 120000 - false - - - false - false - - - false - false - true - - - -""" - -def mangle(tree): - if tree.xpath("//jenkins.plugins.publish__over__ssh.BapSshPublisherPlugin"): - # Already there - return - tag = tree.xpath('/project/publishers')[0] - node = fromstring(publisher) - tag.append(node) - node.getprevious().tail += " " - node.tail = "\n " diff --git a/utils/mangle-jobs/artifacts-add.mangle b/utils/mangle-jobs/artifacts-add.mangle deleted file mode 100644 index 1352135..0000000 --- a/utils/mangle-jobs/artifacts-add.mangle +++ /dev/null @@ -1,19 +0,0 @@ -# Add Artifact archiving config - -from lxml.etree import fromstring - - -new_node = fromstring("""\ - - build/out/*.xml - false - -""") - -def mangle(tree): - if tree.xpath("//hudson.tasks.ArtifactArchiver"): - # Needed config already there - don't change - return False - - parent_node = tree.xpath('/project/publishers')[0] - parent_node.insert(0, new_node) diff --git a/utils/mangle-jobs/artifacts-remove.mangle b/utils/mangle-jobs/artifacts-remove.mangle deleted file mode 100644 index 0ec0810..0000000 --- a/utils/mangle-jobs/artifacts-remove.mangle +++ /dev/null @@ -1,13 +0,0 @@ -# Disable Jenkins builtin artifact archiving (part of migration to snapshots.linaro.org) - -from mangle_helper import * - - -def mangle(tree): - cfg = get_build_config(tree) - if cfg.get("BUILD_TYPE", "build-android") != "build-android": - return - tags = tree.xpath('//hudson.tasks.ArtifactArchiver') - if not tags: - return - tags[0].getparent().remove(tags[0]) diff --git a/utils/mangle-jobs/artifacts-set.mangle b/utils/mangle-jobs/artifacts-set.mangle deleted file mode 100644 index 4a5c707..0000000 --- a/utils/mangle-jobs/artifacts-set.mangle +++ /dev/null @@ -1,15 +0,0 @@ -# Update list of archived artifacts -# Note - since we stopped to use native Jenkins archiving, -# this script is no longer used - -new_value = "build/out/target/*/*/*.img," \ - "build/out/target/*/*/*.tar.bz2," \ - "build/out/target/*/*/MD5SUMS," \ - "build/out/*.tar.bz2," \ - "build/out/*.xml," \ - "build/out/*_config," \ - "build/out/lava-job-info" - -def mangle(tree): - tags = tree.xpath('/project/publishers/hudson.tasks.ArtifactArchiver/artifacts') - tags[0].text = new_value diff --git a/utils/mangle-jobs/build-expiration-set.mangle b/utils/mangle-jobs/build-expiration-set.mangle deleted file mode 100644 index d9d8a6d..0000000 --- a/utils/mangle-jobs/build-expiration-set.mangle +++ /dev/null @@ -1,17 +0,0 @@ -# Set expiration policy for a job - -from lxml.etree import fromstring -from mangle_helper import * - - -new_node = """\ - - 90 - 100 - -1 - -1 - -""" - -def mangle(tree): - add_or_replace_node(tree, "/project/logRotator", new_node) diff --git a/utils/mangle-jobs/build-steps-set.mangle b/utils/mangle-jobs/build-steps-set.mangle deleted file mode 100644 index 16f5ec3..0000000 --- a/utils/mangle-jobs/build-steps-set.mangle +++ /dev/null @@ -1,19 +0,0 @@ -# Set latest build steps config for all jobs, as provided in build.xml file - -from lxml.etree import fromstring - - -builders = open("builders.xml").read() - -def mangle(tree): - tags = tree.xpath("//jenkins.plugins.publish__over__ssh.BapSshPublisherPlugin") - for t in tags: - t.getparent().remove(t) - - # Replace existing content - node = fromstring(builders) - # Add our complete content after the current - tag = tree.xpath('/project/builders')[0] - tag.addnext(node) - # And now remove the original - tag.getparent().remove(tag) diff --git a/utils/mangle-jobs/build-timeout-add.mangle b/utils/mangle-jobs/build-timeout-add.mangle deleted file mode 100644 index 142f70a..0000000 --- a/utils/mangle-jobs/build-timeout-add.mangle +++ /dev/null @@ -1,22 +0,0 @@ -# Add build-timeout plugin setup for build dependencies - -#from lxml.etree import Element -from lxml.etree import fromstring - -new_node = """\ - - 150 - false - -""" - -def mangle(tree): - if tree.xpath("//hudson.plugins.build__timeout.BuildTimeoutWrapper"): - # Already there - return - tag = tree.xpath('/project/buildWrappers')[0] - node = fromstring(new_node) - tag.append(node) - if node.getprevious(): - node.getprevious().tail += " " - node.tail = "\n " diff --git a/utils/mangle-jobs/build-timeout-set.mangle b/utils/mangle-jobs/build-timeout-set.mangle deleted file mode 100644 index 9c4b113..0000000 --- a/utils/mangle-jobs/build-timeout-set.mangle +++ /dev/null @@ -1,15 +0,0 @@ -# Update default build timeout value to new default. -# Don't touch custom timeout times - -OLD_DEFAULT = "275" -NEW_DEFAULT = "375" - - -def mangle(tree): - tags = tree.xpath("//hudson.plugins.build__timeout.BuildTimeoutWrapper/timeoutMinutes") - if not tags: - return False - tag = tags[0] - if tag.text != OLD_DEFAULT: - return False - tag.text = NEW_DEFAULT diff --git a/utils/mangle-jobs/builders.xml b/utils/mangle-jobs/builders.xml deleted file mode 100644 index 4d89266..0000000 --- a/utils/mangle-jobs/builders.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - # Runs as ubuntu -rm -rf build-tools -bzr get lp:linaro-android-build-tools build-tools -sudo -H -E build-tools/node/build us-east-1.ec2-git-mirror.linaro.org "$CONFIG" - -time build-tools/utils/new-publish/publish -p2 $JOB_NAME/$BUILD_NUMBER "build/out/target/*/*/*.img,build/out/target/*/*/*.img.bz2,build/out/target/*/*/*.tar.bz2,build/out/target/*/*/MD5SUMS,build/out/*.tar.bz2,build/out/*.xml,build/out/*_config,build/out/lava-job-info,build/out/linaro_kernel_build_cmds.sh,build/out/linaro_android_build_cmds.sh,build/out/BUILD-INFO.txt,build/out/*/BUILD-INFO.txt,build/out/*/*/BUILD-INFO.txt,build/out/*/*/*/BUILD-INFO.txt,build/out/target/product/*/howto/HOWTO_*.txt,build/out/target/product/*/HOWTO_*.txt" - -build-tools/node/lava-submit "$CONFIG" - -time build-tools/utils/new-publish/publish -p2 $JOB_NAME/$BUILD_NUMBER build/out/lava-job-info - -echo "Build finished" - - - diff --git a/utils/mangle-jobs/ci-tools-checkout.mangle b/utils/mangle-jobs/ci-tools-checkout.mangle deleted file mode 100644 index 46b9b5f..0000000 --- a/utils/mangle-jobs/ci-tools-checkout.mangle +++ /dev/null @@ -1,9 +0,0 @@ -# Update initial startup shell commands for a job - -old_cmd = 'bzr branch lp:~linaro-infrastructure/linaro-ci/lci-build-tools' -new_cmd = 'bzr branch lp:linaro-ci lci-build-tools' - -def mangle(tree): - tags = tree.xpath('/project/builders/hudson.tasks.Shell/command') - if tags: - tags[0].text = tags[0].text.replace(old_cmd, new_cmd) diff --git a/utils/mangle-jobs/copytoslave-remove.mangle b/utils/mangle-jobs/copytoslave-remove.mangle deleted file mode 100644 index 3ca70b5..0000000 --- a/utils/mangle-jobs/copytoslave-remove.mangle +++ /dev/null @@ -1,10 +0,0 @@ -# Remove CopyToSlave plugin config section - -from mangle_helper import * - - -def mangle(tree): - tags = tree.xpath('//com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper') - if not tags: - return - tags[0].getparent().remove(tags[0]) diff --git a/utils/mangle-jobs/cron-set-no-weekend.mangle b/utils/mangle-jobs/cron-set-no-weekend.mangle deleted file mode 100644 index b6091fe..0000000 --- a/utils/mangle-jobs/cron-set-no-weekend.mangle +++ /dev/null @@ -1,12 +0,0 @@ -# Set cron weekend spec to skip Sat - - -def mangle(tree): - nodes = tree.xpath("//hudson.triggers.TimerTrigger/spec") - if not nodes: - return False - spec = nodes[0].text.split() - if spec[-1] != "*": - return False - spec[-1] = "0-5" - nodes[0].text = " ".join(spec) diff --git a/utils/mangle-jobs/description-prepend.mangle b/utils/mangle-jobs/description-prepend.mangle deleted file mode 100644 index 8c08789..0000000 --- a/utils/mangle-jobs/description-prepend.mangle +++ /dev/null @@ -1,28 +0,0 @@ -# This script prepends a notice to each job's description -# (or replaces such notice if already there). - -from mangle_helper import * - - -PREPEND_TEXT = open("header.html").read() - -HEADER = "\n\n" -FOOTER = "" - -def mangle(tree): - cfg = get_build_config(tree) - build_type = cfg.get("BUILD_TYPE") - if not build_type: - build_type = cfg.get("SCRIPT_NAME", "build-android") - if build_type != "build-android": - return - - nodes = tree.xpath('/project/description') - if not nodes: return - descr = nodes[0].text - if descr is None: - descr = "" - if FOOTER in descr: - descr = descr[descr.find(FOOTER) + len(FOOTER):] - descr = HEADER + PREPEND_TEXT + "\n" + FOOTER + "\n\n" + descr - nodes[0].text = descr diff --git a/utils/mangle-jobs/email-ext-notification-set.mangle b/utils/mangle-jobs/email-ext-notification-set.mangle deleted file mode 100644 index 73b0dc0..0000000 --- a/utils/mangle-jobs/email-ext-notification-set.mangle +++ /dev/null @@ -1,20 +0,0 @@ -# -# Update email notification recepient list -# - -from mangle_helper import * - - -builders = 'linaro-android-builder-notifications@linaro.org' -restricted_builders = 'linaro-android-restricted-builder-notifications@linaro.org' - -def mangle(tree): - if not tree.xpath('//hudson.plugins.emailext.ExtendedEmailPublisher'): - # If node doesn't exist do nothing. - return - tags = tree.xpath('/project/publishers/hudson.plugins.emailext.ExtendedEmailPublisher/recipientList') - cfg = get_build_config(tree) - if cfg.get("BUILD_TYPE", "build-android") == "build-android-restricted": - tags[0].text = restricted_builders - else: - tags[0].text = builders diff --git a/utils/mangle-jobs/fingerprinter-add.mangle b/utils/mangle-jobs/fingerprinter-add.mangle deleted file mode 100644 index 61edd24..0000000 --- a/utils/mangle-jobs/fingerprinter-add.mangle +++ /dev/null @@ -1,25 +0,0 @@ -# Add Fingerprinting config - -from lxml.etree import fromstring - - -new_node = fromstring("""\ - - build/fingerprints/* - false - -""") - -def mangle(tree): - if tree.xpath("//hudson.tasks.Fingerprinter"): - # Needed config already there - don't change - return False - - parent_node = tree.xpath('/project/publishers')[0] - parent_node.append(new_node) - - # Adjust whitespacing around a node for pretty formatting - # You may need to update thelines below, based on a test run, - # if you reuse this script for another case. -# new_node.getprevious().tail += " " - new_node.tail = "\n " diff --git a/utils/mangle-jobs/header.html b/utils/mangle-jobs/header.html deleted file mode 100644 index 5ca927a..0000000 --- a/utils/mangle-jobs/header.html +++ /dev/null @@ -1,12 +0,0 @@ - - -

-
-

-New!
-We now provide linaro_android_build_cmds.sh script (see downloads) -with exact commands to reproduce this build. Use it as a reference of -commands you need to execute, or just run to perform a build automatically. -There is also linaro_kernel_build_cmds.sh to rebuild just a kernel. -Alternatively, you can try instructions below. -

diff --git a/utils/mangle-jobs/job-shell-commands-set.mangle b/utils/mangle-jobs/job-shell-commands-set.mangle deleted file mode 100644 index 81abacc..0000000 --- a/utils/mangle-jobs/job-shell-commands-set.mangle +++ /dev/null @@ -1,11 +0,0 @@ -# Update initial startup shell commands for a job - -new_cmd = '''\ -rm -rf build-tools -bzr get lp:linaro-android-build-tools build-tools -sudo -H -E build-tools/node/build us-east-1.ec2-git-mirror.linaro.org "$CONFIG" -''' - -def mangle(tree): - tags = tree.xpath('/project/builders/hudson.tasks.Shell/command') - tags[0].text = new_cmd diff --git a/utils/mangle-jobs/job-shell-commands-set2.mangle b/utils/mangle-jobs/job-shell-commands-set2.mangle deleted file mode 100644 index 5771450..0000000 --- a/utils/mangle-jobs/job-shell-commands-set2.mangle +++ /dev/null @@ -1,11 +0,0 @@ -# Update initial startup shell commands for a job - -new_cmd = '''\ -rm -rf build-tools -bzr get lp:linaro-android-build-tools build-tools -sudo -H -E build-tools/node/build us-east-1.ec2-git-mirror.linaro.org "$CONFIG" -''' - -def mangle(tree): - tags = tree.xpath('/project/builders/hudson.plugins.shell.ShellWithStatus/command') - tags[0].text = new_cmd diff --git a/utils/mangle-jobs/logparser-add.mangle b/utils/mangle-jobs/logparser-add.mangle deleted file mode 100644 index 597a66e..0000000 --- a/utils/mangle-jobs/logparser-add.mangle +++ /dev/null @@ -1,20 +0,0 @@ -# Add Log Parser plugin application to jobs - -#from lxml.etree import Element -from lxml.etree import fromstring - -def mangle(tree): - if tree.xpath("//hudson.plugins.logparser.LogParserPublisher"): - # Already there - return - tag = tree.xpath('//publishers')[0] - node = fromstring("""\ - - false - false - /var/lib/jenkins/userContent/android.parse - -""") - tag.append(node) - node.getprevious().tail += " " - node.tail = "\n " diff --git a/utils/mangle-jobs/mangle-jobs b/utils/mangle-jobs/mangle-jobs deleted file mode 100755 index 9c465e9..0000000 --- a/utils/mangle-jobs/mangle-jobs +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/python -"""Helper to mass-edit jobs in jenkins. - -""" - -############################################################################### -# Copyright (c) 2011 Linaro -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# which accompanies this distribution, and is available at -# http://www.eclipse.org/legal/epl-v10.html -############################################################################### - -import base64 -from contextlib import nested -import json -import os -import sys -import copy -import re -from tempfile import NamedTemporaryFile -import urllib2 -import optparse -import getpass -from xml.dom import minidom - -from lxml.etree import fromstring, tostring - - -optparser = optparse.OptionParser(usage="%prog ") -optparser.add_option("--url", default="http://localhost:8080/jenkins/", - help="Jenkins base url, default: %default") -optparser.add_option("--user", - help="Jenkins username") -optparser.add_option("--passwd-file", metavar="FILE", - help="File holding Jenkins password") -optparser.add_option("--really", action="store_true", - help="Actually perform changes") -optparser.add_option("--filter-jobname", - help="Process only jobs matching regex pattern") -optparser.add_option("--limit", type="int", default=-1, - help="Change at most LIMIT jobs") -optparser.add_option("--file", - help="Process a file instead of all jobs on a remote server") - -options, args = optparser.parse_args(sys.argv[1:]) -if len(args) != 1: - optparser.error("Wrong number of arguments") - -d = {} -execfile(args[0], d, d) -mangler = d['mangle'] - -password = None -if options.passwd_file: - password = open(options.passwd_file).read().strip() -elif not options.file: - password = getpass.getpass("Password/API Token:") - -if options.url[-1] != '/': - options.url += '/' - -auth_headers = { - 'Authorization': 'Basic %s' % ( - base64.encodestring('%s:%s' % (options.user, password))[:-1],), - } - -def _authJenkins(jenkins_path, data=None, extra_headers=None): - """Make an authenticated request to jenkins. - - @param jenkins_path: The path on the Jenkins instance to make the request - to. - @param data: Data to include in the request (if this is not None the - request will be a POST). - @param extra_headers: A dictionary of extra headers that will passed in - addition to Authorization. - @raises urllib2.HTTPError: If the response is not a HTTP 200. - @returns: the body of the response. - """ - headers = auth_headers.copy() - if extra_headers: - headers.update(extra_headers) - req = urllib2.Request( - options.url + jenkins_path, data, headers) - resp = urllib2.urlopen(req) - return resp.read() - -def getJobConfig(job_name): - return _authJenkins('job/' + job_name + '/config.xml') - -def postConfig(url, configXml, extra_headers=None): - headers = {'Content-Type': 'text/xml', } - if extra_headers is not None: - headers.update(extra_headers) - _authJenkins(url, configXml, headers) - -def render_xml(tree): - # Render XML to exact dialect used by Jenkins - # This involves some dirty magic - text = tostring(tree, xml_declaration=True, encoding='UTF-8') - # Roundtrip via minidom, this takes care of encoding " as entities - tree2 = minidom.parseString(text) - text = tree2.toxml('UTF-8') - - # expand empty tags - text = re.sub(r"<([-A-Za-z.]+)/>", "<\\1>", text) - - # Some CR noise should be entities - text = text.replace("\r", " ") - - # Finally, munge xml decl - line1, rest = text.split("><", 1) - line1 = line1.replace('"', "'") - r = line1 + ">\n<" + rest - - return r - -def show_diff(old, new): - with nested(NamedTemporaryFile(), NamedTemporaryFile()) as (a, b): - a.write(old) - b.write(new) - a.flush(); b.flush() - os.system('diff -u %s %s' % (a.name, b.name)) - print - -def indent_tree(elem, level=0): - "Indent XML tree for pretty-printing" - i = "\n" + level*" " - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: - indent_tree(elem, level+1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i - -def normalize2text(tree): - """Return normalized text representation of XML tree, suitable for - diffing with normal diff tool.""" - normalized = copy.deepcopy(tree) - indent_tree(normalized) - return tostring(normalized) - -def match_job_name(job_name): - "Check if job name matches filters which may be specified on command line." - if not options.filter_jobname: - return True - neg = False - r = options.filter_jobname - if r[0] == "-": - neg = True - r = r[1:] - return bool(re.search(r, job_name)) ^ neg - -def get_csrf_token(): - try: - crumb_data = _authJenkins('crumbIssuer/api/xml') - except urllib2.HTTPError: - # Ignore errors for android-build which provides no crumb. - return None - tree = minidom.parseString(crumb_data) - crumb_tag = tree.getElementsByTagName('crumb')[0] - field_tag = tree.getElementsByTagName('crumbRequestField')[0] - crumb = str(crumb_tag.firstChild.wholeText) - field = str(field_tag.firstChild.wholeText) - return (field, crumb) - -def process_remote_jenkins(): - jobs = json.load(urllib2.urlopen(options.url + 'api/json?tree=jobs[name]')) - names = [job['name'] for job in jobs['jobs']] - names = [name for name in names if name == 'blank' or '_' in name] - limit = options.limit - - csrf_token = get_csrf_token() - if csrf_token is None: - extra_headers = None - else: - extra_headers = { csrf_token[0]: csrf_token[1], } - - for name in names: - if not match_job_name(name): - continue - if limit == 0: - break - limit -= 1 - print "Processing:" + name - sys.stdout.flush() - org_text = getJobConfig(name) - tree = fromstring(org_text) - org_normalized = normalize2text(tree) - - if mangler(tree) == False: - continue - - if not options.really: - new_normalized = normalize2text(tree) - show_diff(org_normalized, new_normalized) - else: - new_text = render_xml(tree) - if type(new_text) == type(u""): - new_text = new_text.encode("utf8") - postConfig(str('job/' + name + '/config.xml'), new_text, - extra_headers) - - -def main(): - if options.file: - text = open(options.file).read() - tree = fromstring(text) - - org_normalized = normalize2text(tree) - mangler(tree) - new_normalized = normalize2text(tree) - show_diff(org_normalized, new_normalized) - else: - process_remote_jenkins() - -if __name__ == "__main__": - main() diff --git a/utils/mangle-jobs/mangle_helper.py b/utils/mangle-jobs/mangle_helper.py deleted file mode 100644 index 363d2c2..0000000 --- a/utils/mangle-jobs/mangle_helper.py +++ /dev/null @@ -1,48 +0,0 @@ -import base64 -import lxml.etree - - -def get_build_config(tree): - tag = tree.xpath("/project/properties//defaultValue") - assert len(tag) == 1 - tag = tag[0] - d = {} - for l in base64.decodestring(tag.text).split("\n"): - l = l.strip() - if not l or l[0] == "#": - continue - k, v = l.split("=", 1) - d[k] = v - return d - - -def add_or_replace_node(tree, node_xpath, node_text): - new_node = lxml.etree.fromstring(node_text) - nodes = tree.xpath(node_xpath) - assert len(nodes) < 2, "Please use more selective XPath expression" - if nodes: - # First, add new node after existing - nodes[0].addnext(new_node) - # Then, delete the original node - nodes[0].getparent().remove(nodes[0]) - else: - parent_xpath, _ = node_xpath.rsplit("/", 1) - parent = tree.xpath(parent_xpath)[0] - parent.append(new_node) - - -def add_child(tree, node_xpath, node_text): - "Add new node (node_text) as last child of node_xpath." - nodes = tree.xpath(node_xpath) - assert len(nodes) == 1, "Found %d nodes with XPath %s, expected 1" % (len(nodes), node_xpath) - new_node = lxml.etree.fromstring(node_text) - nodes[0].append(new_node) - - -def add_sibling(tree, node_xpath, node_text): - "Add new node (node_text) as next sibling after node_xpath." - new_node = lxml.etree.fromstring(node_text) - nodes = tree.xpath(node_xpath) - assert len(nodes) > 0, "Node not found: %s" % node_xpath - assert len(nodes) < 2, "Please use more selective XPath expression than %s" % node_xpath - nodes[0].addnext(new_node) diff --git a/utils/mangle-jobs/null.mangle b/utils/mangle-jobs/null.mangle deleted file mode 100644 index e52032e..0000000 --- a/utils/mangle-jobs/null.mangle +++ /dev/null @@ -1,4 +0,0 @@ -# Make no changes, for identity testing - -def mangle(tree): - pass diff --git a/utils/mangle-jobs/project-based-sec-add.mangle b/utils/mangle-jobs/project-based-sec-add.mangle deleted file mode 100644 index d59faaf..0000000 --- a/utils/mangle-jobs/project-based-sec-add.mangle +++ /dev/null @@ -1,17 +0,0 @@ -# Add default project-based security setting for project (anon read access) - -from lxml.etree import fromstring -from mangle_helper import * - -new_node = """\ - - hudson.model.Item.Read:anonymous - -""" - -def mangle(tree): - if tree.xpath("//properties/hudson.security.AuthorizationMatrixProperty"): - # Needed config already there - don't change - return False - - add_child(tree, "/project/properties", new_node) diff --git a/utils/mangle-jobs/push-artifacts-set.mangle b/utils/mangle-jobs/push-artifacts-set.mangle deleted file mode 100644 index 67e0045..0000000 --- a/utils/mangle-jobs/push-artifacts-set.mangle +++ /dev/null @@ -1,26 +0,0 @@ -# -# This script is no longer used! -# -# See build-steps-set.mangle & builders.xml -# - - -new_value = "build/out/target/*/*/*.img," \ - "build/out/target/*/*/*.img.bz2," \ - "build/out/target/*/*/*.tar.bz2," \ - "build/out/target/*/*/MD5SUMS," \ - "build/out/*.tar.bz2," \ - "build/out/*.xml," \ - "build/out/*_config," \ - "build/out/lava-job-info," \ - "build/out/linaro_kernel_build_cmds.sh," \ - "build/out/linaro_android_build_cmds.sh," \ - "build/out/**/BUILD-INFO.txt," \ - "build/out/target/product/*/HOWTO_*.txt" \ - "build/out/target/product/*/howto/HOWTO_*.txt" - -def mangle(tree): - nodes = tree.xpath('//jenkins.plugins.publish__over__ssh.BapSshTransfer/sourceFiles') - # There're few such nodes, ours is first, but extra check to be sure - assert nodes[0].text.startswith('build/out/') - nodes[0].text = new_value diff --git a/utils/mangle-jobs/reset-cron.mangle b/utils/mangle-jobs/reset-cron.mangle deleted file mode 100644 index 3848ac4..0000000 --- a/utils/mangle-jobs/reset-cron.mangle +++ /dev/null @@ -1,8 +0,0 @@ -# Reset any triggers of a job, cron triggers in particular (but not limited) -# This is useful for sandboxes to avoid unexpected cron jobs runs - -def mangle(tree): - tag = tree.xpath('//triggers')[0] - for t in tag: - tag.remove(t) - tag.text = None diff --git a/utils/mangle-jobs/restricted-relabel.mangle b/utils/mangle-jobs/restricted-relabel.mangle deleted file mode 100644 index 131065e..0000000 --- a/utils/mangle-jobs/restricted-relabel.mangle +++ /dev/null @@ -1,11 +0,0 @@ -# Replace labels for restricted jobs - - -def mangle(tree): - nodes = tree.xpath("//assignedNode") - if not nodes: - return False - labels = nodes[0].text - if "private" not in labels: - return False - nodes[0].text = "ec2-restricted" diff --git a/utils/mangle-jobs/shell-status-set.mangle b/utils/mangle-jobs/shell-status-set.mangle deleted file mode 100644 index aab05e6..0000000 --- a/utils/mangle-jobs/shell-status-set.mangle +++ /dev/null @@ -1,11 +0,0 @@ -# Make 1st of 2 shell scripts we have run using shell-status plugin - -#from lxml.etree import Element -from lxml.etree import fromstring - -def mangle(tree): - tags = tree.xpath('//hudson.tasks.Shell') - if len(tags) != 2: - # Already applied - return - tags[0].tag = "hudson.plugins.shell.ShellWithStatus" diff --git a/utils/mangle-jobs/test.mangle b/utils/mangle-jobs/test.mangle deleted file mode 100644 index ea5dd84..0000000 --- a/utils/mangle-jobs/test.mangle +++ /dev/null @@ -1,7 +0,0 @@ -# Sample test mangle script - -def mangle(tree): - tags = tree.xpath('/project/description') - if not tags: return - tag = tags[0] - tag.text = 'Excitingly created.' diff --git a/utils/migrate-artifacts/builds-on-snapshots b/utils/migrate-artifacts/builds-on-snapshots deleted file mode 100644 index 54c1daf..0000000 --- a/utils/migrate-artifacts/builds-on-snapshots +++ /dev/null @@ -1,911 +0,0 @@ -doanac/tracking-panda-linaro4.5/1 -doanac/panda-with-google4.4/6 -doanac/panda-with-google4.4/7 -doanac/tracking-panda-google4.4/3 -doanac/tracking-panda-google4.4/2 -doanac/tracking-panda-google4.4/1 -doanac/panda-with-linaro-4.6/1 -doanac/tracking-panda-linaro4.6/2 -doanac/tracking-panda-linaro4.6/1 -doanac/testBuild-0/3 -doanac/testBuild-0/5 -doanac/testBuild-0/4 -doanac/testBuild-0/2 -liuyq0307/testBuild-1/18 -liuyq0307/testBuild-1/40 -liuyq0307/testBuild-1/34 -liuyq0307/testBuild-1/35 -liuyq0307/testBuild-1/8 -liuyq0307/testBuild-1/15 -liuyq0307/testBuild-1/36 -liuyq0307/testBuild-1/47 -liuyq0307/testBuild-1/16 -liuyq0307/testBuild-1/41 -liuyq0307/testBuild-1/20 -liuyq0307/testBuild-1/10 -liuyq0307/testBuild-1/26 -liuyq0307/testBuild-1/39 -liuyq0307/testBuild-1/14 -liuyq0307/testBuild-1/11 -liuyq0307/testBuild-1/46 -liuyq0307/testBuild-1/24 -liuyq0307/testBuild-1/48 -liuyq0307/testBuild-1/45 -liuyq0307/testBuild-1/32 -liuyq0307/testBuild-1/28 -liuyq0307/testBuild-1/44 -liuyq0307/testBuild-1/13 -liuyq0307/testBuild-1/33 -liuyq0307/testBuild-1/43 -liuyq0307/testBuild-1/25 -liuyq0307/testBuild-1/21 -liuyq0307/testBuild-1/31 -liuyq0307/testBuild-1/17 -liuyq0307/testBuild-1/9 -liuyq0307/testBuild-1/27 -liuyq0307/testBuild-1/42 -liuyq0307/testBuild-1/23 -liuyq0307/testBuild-1/19 -liuyq0307/testBuild-1/30 -liuyq0307/testBuild-1/22 -liuyq0307/testBuild-1/12 -liuyq0307/staging-panda/10 -liuyq0307/staging-panda/3 -liuyq0307/staging-panda/5 -liuyq0307/staging-panda/4 -liuyq0307/staging-panda/6 -liuyq0307/staging-panda/7 -liuyq0307/staging-panda/2 -liuyq0307/staging-panda/9 -liuyq0307/staging-panda/1 -liuyq0307/landing-panda/8 -liuyq0307/landing-panda/10 -liuyq0307/landing-panda/3 -liuyq0307/landing-panda/5 -liuyq0307/landing-panda/4 -liuyq0307/landing-panda/6 -liuyq0307/landing-panda/7 -liuyq0307/landing-panda/2 -liuyq0307/landing-panda/9 -liuyq0307/landing-panda/1 -gerrit-bot/pandaboard/56 -gerrit-bot/pandaboard/57 -danilo/tracking-panda-noramdisk/2 -danilo/overlay-test/1 -danilo/snowball-end-to-end/11 -berolinux/staging-vexpress-a9-4.0.3/3 -berolinux/staging-snowball-4.0.3/2 -berolinux/tracking-panda-4.0.3/2 -berolinux/imx6-test/14 -berolinux/imx6-test/13 -berolinux/staging-panda-4.0.3/3 -berolinux/staging-panda-4.0.3/2 -berolinux/panda-bisect-component/1 -berolinux/landing-snowball-4.0.3/7 -berolinux/staging-imx53-4.0.3/2 -berolinux/landing-panda-4.0.3/18 -berolinux/landing-panda-4.0.3/8 -berolinux/landing-panda-4.0.3/15 -berolinux/landing-panda-4.0.3/14 -berolinux/landing-panda-4.0.3/17 -berolinux/landing-panda-4.0.3/1 -berolinux/staging-origen-4.0.3/4 -pundiramit/testBuild-1/8 -pundiramit/testBuild-1/10 -pundiramit/testBuild-1/3 -pundiramit/testBuild-1/5 -pundiramit/testBuild-1/11 -pundiramit/testBuild-1/4 -pundiramit/testBuild-1/6 -pundiramit/testBuild-1/2 -pundiramit/testBuild-1/9 -pundiramit/testBuild-1/1 -pfefferz/aosp-panda/1 -pfefferz/bisect-staging-panda-140/1 -pfefferz/bisect-imx53-mouse/1 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/180 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/183 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/182 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/181 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/185 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/178 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/179 -linaro-android/imx53-ics-gcc46-freescalelt-stable-open/184 -linaro-android/leb-panda-11.05-release/1 -linaro-android/staging-snowball-11.12-release/4 -linaro-android/staging-snowball-11.12-release/2 -linaro-android/staging-snowball-11.12-release/1 -linaro-android/tracking-panda/142 -linaro-android/tracking-panda/131 -linaro-android/tracking-panda/149 -linaro-android/tracking-panda/156 -linaro-android/tracking-panda/172 -linaro-android/tracking-panda/152 -linaro-android/tracking-panda/170 -linaro-android/tracking-panda/163 -linaro-android/tracking-panda/154 -linaro-android/tracking-panda/157 -linaro-android/tracking-panda/164 -linaro-android/tracking-panda/133 -linaro-android/tracking-panda/167 -linaro-android/tracking-panda/135 -linaro-android/tracking-panda/132 -linaro-android/tracking-panda/173 -linaro-android/tracking-panda/127 -linaro-android/tracking-panda/146 -linaro-android/tracking-panda/128 -linaro-android/tracking-panda/121 -linaro-android/tracking-panda/162 -linaro-android/tracking-panda/145 -linaro-android/tracking-panda/151 -linaro-android/tracking-panda/134 -linaro-android/tracking-panda/141 -linaro-android/tracking-panda/148 -linaro-android/tracking-panda/140 -linaro-android/tracking-panda/123 -linaro-android/tracking-panda/161 -linaro-android/tracking-panda/126 -linaro-android/tracking-panda/171 -linaro-android/tracking-panda/139 -linaro-android/tracking-panda/155 -linaro-android/tracking-panda/160 -linaro-android/tracking-panda/168 -linaro-android/tracking-panda/166 -linaro-android/tracking-panda/125 -linaro-android/tracking-panda/129 -linaro-android/tracking-panda/165 -linaro-android/tracking-panda/120 -linaro-android/tracking-panda/153 -linaro-android/tracking-panda/130 -linaro-android/tracking-panda/169 -linaro-android/tracking-panda/122 -linaro-android/tracking-panda/124 -linaro-android/tracking-panda/147 -linaro-android/tracking-panda/174 -linaro-android/tracking-panda/138 -linaro-android/panda-11.11-release/5 -linaro-android/panda-11.11-release/4 -linaro-android/panda-11.11-release/2 -linaro-android/panda-11.11-release/1 -linaro-android/beagle-11.06-release/2 -linaro-android/panda-master-gcc44-aosp-stable-blob/40 -linaro-android/panda-master-gcc44-aosp-stable-blob/35 -linaro-android/panda-master-gcc44-aosp-stable-blob/38 -linaro-android/panda-master-gcc44-aosp-stable-blob/36 -linaro-android/panda-master-gcc44-aosp-stable-blob/41 -linaro-android/panda-master-gcc44-aosp-stable-blob/39 -linaro-android/panda-master-gcc44-aosp-stable-blob/42 -linaro-android/panda-master-gcc44-aosp-stable-blob/37 -linaro-android/beagle-11.11-release/3 -linaro-android/beagle-11.11-release/5 -linaro-android/beagle-11.11-release/4 -linaro-android/beagle-11.11-release/1 -linaro-android/staging-panda-11.10-release/3 -linaro-android/staging-panda-11.10-release/4 -linaro-android/staging-panda-11.10-release/2 -linaro-android/staging-panda-11.10-release/1 -linaro-android/staging-imx53-11.09-release/1 -linaro-android/staging-snowball-11.09-release/1 -linaro-android/panda-11.07-release/3 -linaro-android/panda-11.07-release/5 -linaro-android/panda-11.07-release/4 -linaro-android/panda-11.07-release/6 -linaro-android/panda-11.07-release/2 -linaro-android/panda-11.07-release/1 -linaro-android/panda-ics-gcc46-tilt-stable-blob/211 -linaro-android/panda-ics-gcc46-tilt-stable-blob/212 -linaro-android/panda-ics-gcc46-tilt-stable-blob/218 -linaro-android/panda-ics-gcc46-tilt-stable-blob/217 -linaro-android/panda-ics-gcc46-tilt-stable-blob/213 -linaro-android/panda-ics-gcc46-tilt-stable-blob/216 -linaro-android/panda-ics-gcc46-tilt-stable-blob/214 -linaro-android/panda-ics-gcc46-tilt-stable-blob/215 -linaro-android/panda-11.04-release/2 -linaro-android/panda-ics-gcc44-aosp-stable-blob/56 -linaro-android/panda-ics-gcc44-aosp-stable-blob/59 -linaro-android/panda-ics-gcc44-aosp-stable-blob/52 -linaro-android/panda-ics-gcc44-aosp-stable-blob/53 -linaro-android/panda-ics-gcc44-aosp-stable-blob/55 -linaro-android/panda-ics-gcc44-aosp-stable-blob/60 -linaro-android/panda-ics-gcc44-aosp-stable-blob/54 -linaro-android/staging-vexpress-a9-11.12-release/3 -linaro-android/staging-vexpress-a9-11.12-release/1 -linaro-android/stage-snowball-11.09-release/3 -linaro-android/stage-snowball-11.09-release/5 -linaro-android/stage-snowball-11.09-release/4 -linaro-android/toolchain-4.6-2012.01/2 -linaro-android/staging-origen-11.12-release/8 -linaro-android/staging-origen-11.12-release/7 -linaro-android/staging-origen-11.12-release/2 -linaro-android/staging-origen-11.12-release/1 -linaro-android/beagle-11.05-release/2 -linaro-android/panda-4.4/34 -linaro-android/panda-4.4/35 -linaro-android/panda-4.4/38 -linaro-android/panda-4.4/36 -linaro-android/panda-4.4/47 -linaro-android/panda-4.4/41 -linaro-android/panda-4.4/39 -linaro-android/panda-4.4/46 -linaro-android/panda-4.4/48 -linaro-android/panda-4.4/45 -linaro-android/panda-4.4/28 -linaro-android/panda-4.4/44 -linaro-android/panda-4.4/33 -linaro-android/panda-4.4/51 -linaro-android/panda-4.4/43 -linaro-android/panda-4.4/31 -linaro-android/panda-4.4/50 -linaro-android/panda-4.4/27 -linaro-android/panda-4.4/42 -linaro-android/panda-4.4/49 -linaro-android/panda-4.4/37 -linaro-android/staging-panda-11.12-release/5 -linaro-android/staging-panda-11.12-release/4 -linaro-android/staging-panda-11.12-release/6 -linaro-android/staging-panda-11.12-release/7 -linaro-android/staging-panda-11.12-release/2 -linaro-android/staging-panda-11.12-release/1 -linaro-android/staging-vexpress-a9-12.01-release/3 -linaro-android/staging-vexpress-a9-12.01-release/1 -linaro-android/landing-snowball-12.01-release/3 -linaro-android/landing-snowball-12.01-release/5 -linaro-android/landing-snowball-12.01-release/2 -linaro-android/landing-snowball-12.01-release/1 -linaro-android/toolchain-4.5-bzr/56 -linaro-android/toolchain-4.5-bzr/58 -linaro-android/toolchain-4.5-bzr/47 -linaro-android/toolchain-4.5-bzr/59 -linaro-android/toolchain-4.5-bzr/61 -linaro-android/toolchain-4.5-bzr/46 -linaro-android/toolchain-4.5-bzr/48 -linaro-android/toolchain-4.5-bzr/45 -linaro-android/toolchain-4.5-bzr/57 -linaro-android/toolchain-4.5-bzr/53 -linaro-android/toolchain-4.5-bzr/60 -linaro-android/toolchain-4.5-bzr/49 -linaro-android/toolchain-4.5-bzr/54 -linaro-android/toolchain-4.6-bzr/131 -linaro-android/toolchain-4.6-bzr/149 -linaro-android/toolchain-4.6-bzr/143 -linaro-android/toolchain-4.6-bzr/156 -linaro-android/toolchain-4.6-bzr/163 -linaro-android/toolchain-4.6-bzr/154 -linaro-android/toolchain-4.6-bzr/157 -linaro-android/toolchain-4.6-bzr/164 -linaro-android/toolchain-4.6-bzr/144 -linaro-android/toolchain-4.6-bzr/173 -linaro-android/toolchain-4.6-bzr/127 -linaro-android/toolchain-4.6-bzr/146 -linaro-android/toolchain-4.6-bzr/128 -linaro-android/toolchain-4.6-bzr/121 -linaro-android/toolchain-4.6-bzr/162 -linaro-android/toolchain-4.6-bzr/145 -linaro-android/toolchain-4.6-bzr/118 -linaro-android/toolchain-4.6-bzr/148 -linaro-android/toolchain-4.6-bzr/140 -linaro-android/toolchain-4.6-bzr/123 -linaro-android/toolchain-4.6-bzr/161 -linaro-android/toolchain-4.6-bzr/126 -linaro-android/toolchain-4.6-bzr/158 -linaro-android/toolchain-4.6-bzr/139 -linaro-android/toolchain-4.6-bzr/155 -linaro-android/toolchain-4.6-bzr/160 -linaro-android/toolchain-4.6-bzr/159 -linaro-android/toolchain-4.6-bzr/125 -linaro-android/toolchain-4.6-bzr/129 -linaro-android/toolchain-4.6-bzr/117 -linaro-android/toolchain-4.6-bzr/119 -linaro-android/toolchain-4.6-bzr/120 -linaro-android/toolchain-4.6-bzr/153 -linaro-android/toolchain-4.6-bzr/130 -linaro-android/toolchain-4.6-bzr/122 -linaro-android/toolchain-4.6-bzr/124 -linaro-android/toolchain-4.6-bzr/147 -linaro-android/toolchain-4.6-bzr/174 -linaro-android/tracking-panda-11.10-release/5 -linaro-android/tracking-panda-11.10-release/4 -linaro-android/tracking-panda-11.10-release/1 -linaro-android/leb-panda-11.08-release/3 -linaro-android/leb-panda-11.08-release/4 -linaro-android/leb-panda-11.08-release/2 -linaro-android/leb-panda-11.08-release/1 -linaro-android/staging-panda-11.11-release/3 -linaro-android/staging-panda-11.11-release/5 -linaro-android/staging-panda-11.11-release/4 -linaro-android/staging-panda-11.11-release/2 -linaro-android/staging-panda-11.11-release/1 -linaro-android/leb-origen-11.08-release/3 -linaro-android/leb-origen-11.08-release/5 -linaro-android/leb-origen-11.08-release/4 -linaro-android/leb-origen-11.08-release/2 -linaro-android/leb-origen-11.08-release/1 -linaro-android/leb-imx53-11.08-release/1 -linaro-android/staging-snowball-11.11-release/2 -linaro-android/staging-snowball-11.11-release/1 -linaro-android/panda/478 -linaro-android/panda/461 -linaro-android/panda/451 -linaro-android/panda/442 -linaro-android/panda/447 -linaro-android/panda/470 -linaro-android/panda/479 -linaro-android/panda/472 -linaro-android/panda/441 -linaro-android/panda/490 -linaro-android/panda/484 -linaro-android/panda/491 -linaro-android/panda/480 -linaro-android/panda/449 -linaro-android/panda/450 -linaro-android/panda/477 -linaro-android/panda/464 -linaro-android/panda/444 -linaro-android/panda/458 -linaro-android/panda/469 -linaro-android/panda/455 -linaro-android/panda/481 -linaro-android/panda/474 -linaro-android/panda/482 -linaro-android/panda/446 -linaro-android/panda/486 -linaro-android/panda/476 -linaro-android/panda/454 -linaro-android/panda/487 -linaro-android/panda/456 -linaro-android/panda/448 -linaro-android/panda/462 -linaro-android/panda/473 -linaro-android/panda/488 -linaro-android/panda/457 -linaro-android/panda/468 -linaro-android/panda/465 -linaro-android/panda/453 -linaro-android/panda/471 -linaro-android/panda/483 -linaro-android/panda/463 -linaro-android/panda/485 -linaro-android/panda/443 -linaro-android/panda/489 -linaro-android/panda/467 -linaro-android/panda/452 -linaro-android/panda/445 -linaro-android/panda-ics-gcc46-kwg-upstream-open/502 -linaro-android/panda-ics-gcc46-kwg-upstream-open/498 -linaro-android/panda-ics-gcc46-kwg-upstream-open/497 -linaro-android/panda-ics-gcc46-kwg-upstream-open/499 -linaro-android/panda-ics-gcc46-kwg-upstream-open/501 -linaro-android/panda-ics-gcc46-kwg-upstream-open/495 -linaro-android/panda-ics-gcc46-kwg-upstream-open/493 -linaro-android/panda-ics-gcc46-kwg-upstream-open/494 -linaro-android/panda-ics-gcc46-kwg-upstream-open/500 -linaro-android/panda-ics-gcc46-kwg-upstream-open/496 -linaro-android/panda-ics-gcc46-kwg-upstream-open/492 -linaro-android/beagle/342 -linaro-android/beagle/343 -linaro-android/beagle-11.08-release/2 -linaro-android/landing-snowball/142 -linaro-android/landing-snowball/131 -linaro-android/landing-snowball/150 -linaro-android/landing-snowball/149 -linaro-android/landing-snowball/143 -linaro-android/landing-snowball/108 -linaro-android/landing-snowball/113 -linaro-android/landing-snowball/144 -linaro-android/landing-snowball/133 -linaro-android/landing-snowball/135 -linaro-android/landing-snowball/132 -linaro-android/landing-snowball/127 -linaro-android/landing-snowball/106 -linaro-android/landing-snowball/146 -linaro-android/landing-snowball/121 -linaro-android/landing-snowball/101 -linaro-android/landing-snowball/109 -linaro-android/landing-snowball/114 -linaro-android/landing-snowball/145 -linaro-android/landing-snowball/107 -linaro-android/landing-snowball/110 -linaro-android/landing-snowball/99 -linaro-android/landing-snowball/134 -linaro-android/landing-snowball/100 -linaro-android/landing-snowball/118 -linaro-android/landing-snowball/141 -linaro-android/landing-snowball/104 -linaro-android/landing-snowball/105 -linaro-android/landing-snowball/148 -linaro-android/landing-snowball/112 -linaro-android/landing-snowball/126 -linaro-android/landing-snowball/136 -linaro-android/landing-snowball/139 -linaro-android/landing-snowball/125 -linaro-android/landing-snowball/97 -linaro-android/landing-snowball/129 -linaro-android/landing-snowball/117 -linaro-android/landing-snowball/130 -linaro-android/landing-snowball/147 -linaro-android/landing-snowball/103 -linaro-android/landing-snowball/138 -linaro-android/landing-snowball/115 -linaro-android/landing-snowball-11.11-release/3 -linaro-android/landing-snowball-11.11-release/1 -linaro-android/staging-vexpress-a9-11.11-release/1 -linaro-android/leb-panda-11.07-release/3 -linaro-android/leb-panda-11.07-release/5 -linaro-android/leb-panda-11.07-release/4 -linaro-android/panda-ics-gcc46-omapzoom-stable-blob/58 -linaro-android/panda-ics-gcc46-omapzoom-stable-blob/61 -linaro-android/panda-ics-gcc46-omapzoom-stable-blob/57 -linaro-android/panda-ics-gcc46-omapzoom-stable-blob/55 -linaro-android/panda-ics-gcc46-omapzoom-stable-blob/60 -linaro-android/panda-ics-gcc46-omapzoom-stable-blob/54 -linaro-android/staging-panda/189 -linaro-android/staging-panda/194 -linaro-android/staging-panda/191 -linaro-android/staging-panda/172 -linaro-android/staging-panda/183 -linaro-android/staging-panda/210 -linaro-android/staging-panda/201 -linaro-android/staging-panda/199 -linaro-android/staging-panda/170 -linaro-android/staging-panda/206 -linaro-android/staging-panda/176 -linaro-android/staging-panda/182 -linaro-android/staging-panda/164 -linaro-android/staging-panda/185 -linaro-android/staging-panda/178 -linaro-android/staging-panda/175 -linaro-android/staging-panda/167 -linaro-android/staging-panda/177 -linaro-android/staging-panda/173 -linaro-android/staging-panda/209 -linaro-android/staging-panda/204 -linaro-android/staging-panda/202 -linaro-android/staging-panda/186 -linaro-android/staging-panda/188 -linaro-android/staging-panda/179 -linaro-android/staging-panda/205 -linaro-android/staging-panda/196 -linaro-android/staging-panda/193 -linaro-android/staging-panda/190 -linaro-android/staging-panda/203 -linaro-android/staging-panda/171 -linaro-android/staging-panda/195 -linaro-android/staging-panda/184 -linaro-android/staging-panda/168 -linaro-android/staging-panda/166 -linaro-android/staging-panda/208 -linaro-android/staging-panda/165 -linaro-android/staging-panda/198 -linaro-android/staging-panda/169 -linaro-android/staging-panda/207 -linaro-android/staging-panda/192 -linaro-android/staging-panda/174 -linaro-android/beagle-11.10-release/3 -linaro-android/beagle-11.10-release/4 -linaro-android/beagle-11.10-release/2 -linaro-android/beagle-11.10-release/1 -linaro-android/staging-imx53-11.10-release/3 -linaro-android/staging-imx53-11.10-release/4 -linaro-android/staging-imx53-11.10-release/2 -linaro-android/staging-imx53-11.10-release/1 -linaro-android/panda-11.08-release/3 -linaro-android/panda-11.08-release/4 -linaro-android/panda-11.08-release/2 -linaro-android/panda-11.08-release/1 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/87 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/89 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/88 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/90 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/84 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/86 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/83 -linaro-android/vexpress-ics-gcc46-armlt-stable-open/85 -linaro-android/staging-imx53/142 -linaro-android/staging-imx53/131 -linaro-android/staging-imx53/150 -linaro-android/staging-imx53/149 -linaro-android/staging-imx53/143 -linaro-android/staging-imx53/156 -linaro-android/staging-imx53/172 -linaro-android/staging-imx53/152 -linaro-android/staging-imx53/170 -linaro-android/staging-imx53/157 -linaro-android/staging-imx53/176 -linaro-android/staging-imx53/164 -linaro-android/staging-imx53/144 -linaro-android/staging-imx53/175 -linaro-android/staging-imx53/167 -linaro-android/staging-imx53/177 -linaro-android/staging-imx53/135 -linaro-android/staging-imx53/132 -linaro-android/staging-imx53/173 -linaro-android/staging-imx53/127 -linaro-android/staging-imx53/128 -linaro-android/staging-imx53/121 -linaro-android/staging-imx53/162 -linaro-android/staging-imx53/145 -linaro-android/staging-imx53/134 -linaro-android/staging-imx53/141 -linaro-android/staging-imx53/148 -linaro-android/staging-imx53/123 -linaro-android/staging-imx53/126 -linaro-android/staging-imx53/158 -linaro-android/staging-imx53/171 -linaro-android/staging-imx53/160 -linaro-android/staging-imx53/168 -linaro-android/staging-imx53/125 -linaro-android/staging-imx53/129 -linaro-android/staging-imx53/165 -linaro-android/staging-imx53/120 -linaro-android/staging-imx53/153 -linaro-android/staging-imx53/130 -linaro-android/staging-imx53/169 -linaro-android/staging-imx53/122 -linaro-android/staging-imx53/124 -linaro-android/staging-imx53/174 -linaro-android/landing-snowball-11.10-release/3 -linaro-android/landing-snowball-11.10-release/5 -linaro-android/landing-snowball-11.10-release/4 -linaro-android/landing-snowball-11.10-release/2 -linaro-android/landing-snowball-11.10-release/1 -linaro-android/staging-vexpress-a9/66 -linaro-android/staging-vexpress-a9/40 -linaro-android/staging-vexpress-a9/34 -linaro-android/staging-vexpress-a9/35 -linaro-android/staging-vexpress-a9/73 -linaro-android/staging-vexpress-a9/38 -linaro-android/staging-vexpress-a9/36 -linaro-android/staging-vexpress-a9/56 -linaro-android/staging-vexpress-a9/75 -linaro-android/staging-vexpress-a9/58 -linaro-android/staging-vexpress-a9/47 -linaro-android/staging-vexpress-a9/29 -linaro-android/staging-vexpress-a9/80 -linaro-android/staging-vexpress-a9/63 -linaro-android/staging-vexpress-a9/72 -linaro-android/staging-vexpress-a9/39 -linaro-android/staging-vexpress-a9/64 -linaro-android/staging-vexpress-a9/82 -linaro-android/staging-vexpress-a9/68 -linaro-android/staging-vexpress-a9/59 -linaro-android/staging-vexpress-a9/61 -linaro-android/staging-vexpress-a9/46 -linaro-android/staging-vexpress-a9/69 -linaro-android/staging-vexpress-a9/48 -linaro-android/staging-vexpress-a9/32 -linaro-android/staging-vexpress-a9/57 -linaro-android/staging-vexpress-a9/44 -linaro-android/staging-vexpress-a9/33 -linaro-android/staging-vexpress-a9/71 -linaro-android/staging-vexpress-a9/43 -linaro-android/staging-vexpress-a9/62 -linaro-android/staging-vexpress-a9/31 -linaro-android/staging-vexpress-a9/55 -linaro-android/staging-vexpress-a9/42 -linaro-android/staging-vexpress-a9/60 -linaro-android/staging-vexpress-a9/49 -linaro-android/staging-vexpress-a9/30 -linaro-android/staging-vexpress-a9/74 -linaro-android/staging-vexpress-a9/65 -linaro-android/staging-vexpress-a9/37 -linaro-android/landing-snowball-11.12-release/3 -linaro-android/landing-snowball-11.12-release/1 -linaro-android/landing-panda-12.01-release/3 -linaro-android/landing-panda-12.01-release/4 -linaro-android/landing-panda-12.01-release/1 -linaro-android/staging-snowball-11.10-release/3 -linaro-android/staging-snowball-11.10-release/6 -linaro-android/staging-snowball-11.10-release/7 -linaro-android/staging-snowball-11.10-release/2 -linaro-android/staging-snowball-11.10-release/1 -linaro-android/beagle-11.04-release/2 -linaro-android/staging-origen-11.11-release/3 -linaro-android/staging-origen-11.11-release/1 -linaro-android/beagle-11.07-release/3 -linaro-android/beagle-11.07-release/4 -linaro-android/panda-11.10-release/3 -linaro-android/panda-11.10-release/2 -linaro-android/panda-11.10-release/1 -linaro-android/panda-11.09-release/3 -linaro-android/panda-11.09-release/4 -linaro-android/panda-11.09-release/2 -linaro-android/panda-11.09-release/1 -linaro-android/aosp-panda/18 -linaro-android/aosp-panda/34 -linaro-android/aosp-panda/29 -linaro-android/aosp-panda/20 -linaro-android/aosp-panda/26 -linaro-android/aosp-panda/24 -linaro-android/aosp-panda/32 -linaro-android/aosp-panda/28 -linaro-android/aosp-panda/33 -linaro-android/aosp-panda/25 -linaro-android/aosp-panda/21 -linaro-android/aosp-panda/31 -linaro-android/aosp-panda/27 -linaro-android/aosp-panda/23 -linaro-android/aosp-panda/19 -linaro-android/aosp-panda/30 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/180 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/176 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/182 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/181 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/178 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/175 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/177 -linaro-android/panda-ics-gcc46-tilt-tracking-blob/179 -linaro-android/staging-snowball-12.01-release/1 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/172 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/170 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/176 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/178 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/177 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/173 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/171 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/169 -linaro-android/origen-ics-gcc46-samsunglt-stable-blob/174 -linaro-android/leb-snowball-11.08-release/3 -linaro-android/leb-snowball-11.08-release/4 -linaro-android/leb-snowball-11.08-release/2 -linaro-android/leb-snowball-11.08-release/1 -linaro-android/panda-11.12-release/3 -linaro-android/panda-11.12-release/5 -linaro-android/panda-11.12-release/4 -linaro-android/panda-11.12-release/6 -linaro-android/panda-11.12-release/7 -linaro-android/panda-11.12-release/2 -linaro-android/panda-11.12-release/1 -linaro-android/staging-panda-12.01-release/2 -linaro-android/staging-panda-12.01-release/1 -linaro-android/landing-panda-11.12-release/3 -linaro-android/landing-panda-11.12-release/1 -linaro-android/leb-panda-11.06-release/3 -linaro-android/tracking-panda-11.11-release/3 -linaro-android/tracking-panda-11.11-release/4 -linaro-android/tracking-panda-11.11-release/2 -linaro-android/tracking-panda-11.11-release/1 -linaro-android/staging-origen/142 -linaro-android/staging-origen/150 -linaro-android/staging-origen/149 -linaro-android/staging-origen/143 -linaro-android/staging-origen/137 -linaro-android/staging-origen/108 -linaro-android/staging-origen/163 -linaro-android/staging-origen/154 -linaro-android/staging-origen/157 -linaro-android/staging-origen/102 -linaro-android/staging-origen/113 -linaro-android/staging-origen/164 -linaro-android/staging-origen/144 -linaro-android/staging-origen/167 -linaro-android/staging-origen/127 -linaro-android/staging-origen/106 -linaro-android/staging-origen/146 -linaro-android/staging-origen/128 -linaro-android/staging-origen/121 -linaro-android/staging-origen/162 -linaro-android/staging-origen/109 -linaro-android/staging-origen/114 -linaro-android/staging-origen/145 -linaro-android/staging-origen/151 -linaro-android/staging-origen/100 -linaro-android/staging-origen/118 -linaro-android/staging-origen/141 -linaro-android/staging-origen/104 -linaro-android/staging-origen/105 -linaro-android/staging-origen/148 -linaro-android/staging-origen/123 -linaro-android/staging-origen/161 -linaro-android/staging-origen/112 -linaro-android/staging-origen/126 -linaro-android/staging-origen/158 -linaro-android/staging-origen/116 -linaro-android/staging-origen/155 -linaro-android/staging-origen/160 -linaro-android/staging-origen/168 -linaro-android/staging-origen/166 -linaro-android/staging-origen/159 -linaro-android/staging-origen/111 -linaro-android/staging-origen/125 -linaro-android/staging-origen/129 -linaro-android/staging-origen/117 -linaro-android/staging-origen/165 -linaro-android/staging-origen/119 -linaro-android/staging-origen/120 -linaro-android/staging-origen/130 -linaro-android/staging-origen/122 -linaro-android/staging-origen/124 -linaro-android/staging-origen/147 -linaro-android/staging-origen/103 -linaro-android/staging-origen/138 -linaro-android/staging-origen/115 -linaro-android/tracking-panda-12.01-release/3 -linaro-android/tracking-panda-12.01-release/5 -linaro-android/tracking-panda-12.01-release/2 -linaro-android/tracking-panda-12.01-release/1 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/156 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/152 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/154 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/157 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/151 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/158 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/155 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/159 -linaro-android/snowball-ics-gcc46-igloo-stable-blob/153 -linaro-android/staging-imx53-11.12-release/5 -linaro-android/staging-imx53-11.12-release/4 -linaro-android/staging-imx53-11.12-release/6 -linaro-android/staging-imx53-11.12-release/7 -linaro-android/staging-imx53-11.12-release/2 -linaro-android/staging-imx53-11.12-release/1 -linaro-android/landing-panda/40 -linaro-android/landing-panda/35 -linaro-android/landing-panda/15 -linaro-android/landing-panda/36 -linaro-android/landing-panda/47 -linaro-android/landing-panda/16 -linaro-android/landing-panda/29 -linaro-android/landing-panda/20 -linaro-android/landing-panda/10 -linaro-android/landing-panda/3 -linaro-android/landing-panda/39 -linaro-android/landing-panda/14 -linaro-android/landing-panda/11 -linaro-android/landing-panda/4 -linaro-android/landing-panda/46 -linaro-android/landing-panda/6 -linaro-android/landing-panda/48 -linaro-android/landing-panda/45 -linaro-android/landing-panda/32 -linaro-android/landing-panda/44 -linaro-android/landing-panda/7 -linaro-android/landing-panda/52 -linaro-android/landing-panda/13 -linaro-android/landing-panda/33 -linaro-android/landing-panda/51 -linaro-android/landing-panda/43 -linaro-android/landing-panda/53 -linaro-android/landing-panda/21 -linaro-android/landing-panda/31 -linaro-android/landing-panda/2 -linaro-android/landing-panda/17 -linaro-android/landing-panda/50 -linaro-android/landing-panda/42 -linaro-android/landing-panda/49 -linaro-android/landing-panda/23 -linaro-android/landing-panda/19 -linaro-android/landing-panda/30 -linaro-android/landing-panda/1 -linaro-android/landing-panda/22 -linaro-android/leb-imx53-11.07-release/3 -linaro-android/leb-imx53-11.07-release/4 -linaro-android/leb-imx53-11.07-release/2 -linaro-android/panda-12.01-release/2 -linaro-android/panda-12.01-release/1 -linaro-android/tracking-panda-11.12-release/5 -linaro-android/tracking-panda-11.12-release/4 -linaro-android/tracking-panda-11.12-release/6 -linaro-android/tracking-panda-11.12-release/7 -linaro-android/tracking-panda-11.12-release/2 -linaro-android/tracking-panda-11.12-release/1 -linaro-android/staging-origen-11.10-release/5 -linaro-android/staging-origen-11.10-release/4 -linaro-android/staging-origen-11.10-release/2 -linaro-android/staging-origen-11.10-release/1 -linaro-android/stage-origen-11.09-release/3 -linaro-android/stage-origen-11.09-release/2 -linaro-android/stage-origen-11.09-release/1 -linaro-android/staging-panda-11.09-release/1 -linaro-android/leb-snowball-11.07-release/3 -linaro-android/leb-snowball-11.07-release/4 -linaro-android/leb-snowball-11.07-release/2 -linaro-android/leb-snowball-11.07-release/1 -linaro-android/staging-origen-12.01-release/2 -linaro-android/staging-origen-12.01-release/1 -linaro-android/stage-imx53-11.09-release/3 -linaro-android/stage-imx53-11.09-release/2 -linaro-android/stage-imx53-11.09-release/1 -linaro-android/panda-11.06-release/3 -linaro-android/panda-11.06-release/1 -linaro-android/staging-imx53-12.01-release/3 -linaro-android/staging-imx53-12.01-release/6 -linaro-android/staging-imx53-12.01-release/2 -linaro-android/staging-imx53-12.01-release/1 -linaro-android/panda-11.05-release/3 -linaro-android/panda-11.05-release/2 -linaro-android/panda-11.05-release/1 -linaro-android/stage-panda-11.09-release/3 -linaro-android/stage-panda-11.09-release/2 -linaro-android/stage-panda-11.09-release/1 -linaro-android/beagle-11.09-release/3 -linaro-android/beagle-11.09-release/4 -linaro-android/staging-origen-11.09-release/2 -linaro-android/staging-imx53-11.11-release/2 -linaro-android/staging-imx53-11.11-release/1 -linaro-android/staging-snowball/142 -linaro-android/staging-snowball/150 -linaro-android/staging-snowball/149 -linaro-android/staging-snowball/143 -linaro-android/staging-snowball/156 -linaro-android/staging-snowball/172 -linaro-android/staging-snowball/137 -linaro-android/staging-snowball/152 -linaro-android/staging-snowball/170 -linaro-android/staging-snowball/163 -linaro-android/staging-snowball/154 -linaro-android/staging-snowball/157 -linaro-android/staging-snowball/164 -linaro-android/staging-snowball/144 -linaro-android/staging-snowball/167 -linaro-android/staging-snowball/135 -linaro-android/staging-snowball/173 -linaro-android/staging-snowball/146 -linaro-android/staging-snowball/162 -linaro-android/staging-snowball/145 -linaro-android/staging-snowball/141 -linaro-android/staging-snowball/148 -linaro-android/staging-snowball/140 -linaro-android/staging-snowball/161 -linaro-android/staging-snowball/136 -linaro-android/staging-snowball/171 -linaro-android/staging-snowball/139 -linaro-android/staging-snowball/160 -linaro-android/staging-snowball/168 -linaro-android/staging-snowball/159 -linaro-android/staging-snowball/165 -linaro-android/staging-snowball/153 -linaro-android/staging-snowball/147 -linaro-android/staging-snowball/138 -pfalcon/anti-race-prod/1 -pfalcon/panda-uniseed/3 -pfalcon/panda-uniseed/5 -pfalcon/panda-uniseed/4 -pfalcon/panda-uniseed/2 -pfalcon/snapshots-publish/15 -pfalcon/snapshots-publish/13 -pfalcon/snapshots-publish/12 -pfalcon/staging-imx53-uniseed/2 -pfalcon/staging-imx53-uniseed/1 -pfalcon/beagle-end-to-end/3 -pfalcon/anti-race/8 -pfalcon/anti-race/10 -pfalcon/anti-race/3 -pfalcon/anti-race/5 -pfalcon/anti-race/11 -pfalcon/anti-race/6 -pfalcon/anti-race/7 -pfalcon/anti-race/2 -pfalcon/anti-race/9 -pfalcon/anti-race/1 -pfalcon/anti-race/12 -pfalcon/slave-disk-space/8 -pfalcon/slave-disk-space/3 -pfalcon/slave-disk-space/5 -pfalcon/slave-disk-space/4 -pfalcon/slave-disk-space/2 -pfalcon/slave-disk-space/1 -pfalcon/staging-vexpress-a9-uniseed/3 -pfalcon/staging-vexpress-a9-uniseed/2 -pfalcon/staging-vexpress-a9-uniseed/1 -pfalcon/lava-submit-new-urls/2 -pfalcon/lava-submit-new-urls/1 -pfalcon/landing-snowball-uniseed/3 -pfalcon/landing-snowball-uniseed/2 -pfalcon/landing-snowball-uniseed/1 -pfalcon/lava-job-info-transfer/15 -pfalcon/lava-job-info-transfer/16 -pfalcon/lava-job-info-transfer/14 -pfalcon/lava-job-info-transfer/17 -pfalcon/lava-job-info-transfer/1 -pfalcon/staganing-panda-lava-reintegr/1 -austinxxh-linux/testBuild-0/1 -vishalbhoj/tracking-4.0.3/3 -vishalbhoj/tracking-4.0.3/5 -vishalbhoj/tracking-4.0.3/4 -vishalbhoj/tracking-4.0.3/6 -vishalbhoj/tracking-4.0.3/2 -vishalbhoj/tracking-4.0.3/1 -vishalbhoj/landing-panda.xml/2 -vishalbhoj/landing-panda.xml/1 -vishalbhoj/landing-panda-4.0.3/2 -vishalbhoj/landing-panda-4.0.3/1 diff --git a/utils/migrate-artifacts/config.py b/utils/migrate-artifacts/config.py deleted file mode 100755 index 227018e..0000000 --- a/utils/migrate-artifacts/config.py +++ /dev/null @@ -1,18 +0,0 @@ -host = "mombin.canonical.com" -sync_key_file = '/home/ubuntu/snapshots-sync/snapshots-sync' -move_key_file = '/home/ubuntu/snapshots-sync/snapshots-filemove' -sync_user = 'android-build-linaro' -move_user = 'android-build-linaro-trigger' -move_cmd = 'reshuffle-files' -jobs_path = '/var/lib/jenkins/jobs' -jobs_list = ['linaro-android*-release'] -remote_prefix = 'android/.tmp/' -source_files = ['build/out/target/*/*/*.img', - 'build/out/target/*/*/*.tar.bz2', - 'build/out/target/*/*/MD5SUMS', - 'build/out/*.tar.bz2', - 'build/out/*.xml', - 'build/out/*_config', - 'build/out/lava-job-info', - 'build/out/download_and_build.sh'] -remove = '(/archive/build/out)|(/builds)' diff --git a/utils/migrate-artifacts/sync-builds.py b/utils/migrate-artifacts/sync-builds.py deleted file mode 100755 index bb32702..0000000 --- a/utils/migrate-artifacts/sync-builds.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python - -# Script copy old artifacts not present on snapshots.linaro.org from -# android-build.linaro.org like jenkins does via sftp and then moves -# to the desired destination using reshuffle-files command using ssh -# on snapshots.linaro.org -# Artifacts present on s.l.o are taken from list in builds-on-snapshots -# file. -# Command to generate artifacts list on s.l.o: -# find /srv3/snapshots.linaro.org/www/android/ -type d -maxdepth 3 -# -mindepth 3 -wholename '*/~*/*' | sed -# 's/\/srv3\/snapshots\.linaro\.org\/www\/android\/~//' - - -import os -import glob -import re -import tempfile -import subprocess - -from config import * - -builds_list = [] -remove_prefix = re.compile(remove) - - -def create_path_list(path, path_to_create): - while len(path) > 0: - path_to_create.insert(0, path) - head, tail = os.path.split(path) - if len(tail.strip()) == 0: # Just in case path ends with a / or \ - path = head - head, tail = os.path.split(path) - path = head - - return path_to_create - - -def do_sftp_transfer(build, path_to_create, files): - fd, fpath = tempfile.mkstemp() - fnull = open(os.devnull, 'w') - - for path in path_to_create: - os.write(fd, '-mkdir ' + remote_prefix + str(path) + '\n') - for rfile in files: - os.write(fd, '-put ' + str(rfile) + ' ' + remote_prefix + - os.path.dirname(remove_prefix.sub('', str(rfile))) + '\n') - os.close(fd) - subprocess.call(["sftp", "-i", sync_key_file, "-q", "-r", "-b", fpath, - "%s@%s" % (sync_user, host)], stdout = fnull, stderr = fnull) - subprocess.call(["ssh", "-i", move_key_file, "%s@%s" % (move_user, host), - move_cmd, build]) - - fnull.close() - os.remove(fpath) - - -def check_remote(job, build_number): - global builds_list - - return "%s/%s" % (job, build_number) in builds_list - - -def load_build_list(fname): - builds_list = [] - try: - f = open(fname) - except IOError: - raise - else: - with f: - while 1: - line = f.readline() - if not line: - break - builds_list.append(line.replace("/", "_", 1).strip("\n")) - - return builds_list - - -def move_files(job): - for dirname, dirnames, filenames in os.walk('%s/builds' % job): - for build_number in dirnames: - if os.path.islink(os.path.join(dirname, build_number)): - build_started = "%s/%s" % (job, build_number) - if check_remote(job, build_number): - print "Build skipped: " + build_started - continue - files = [] # List of local files to be transferred - remote_paths = [] # List of full remote paths to be created - path_to_create = [] # List of subpaths of full paths - print "Build started: " + build_started - for src in source_files: - files.extend(glob.glob("%s/archive/%s" % - (os.path.join(dirname, build_number), src))) - for rfile in files: - remote_paths.append(remove_prefix.sub('', - os.path.dirname(rfile))) - uniq_remote_paths = list(set(remote_paths)) - for i in uniq_remote_paths: - path_to_create = create_path_list(i, path_to_create) - do_sftp_transfer(build_started, path_to_create, files) - - -if __name__ == '__main__': - builds_list = load_build_list("./builds-on-snapshots") - cwd = os.getcwd() - os.chdir(jobs_path) - for ejob in jobs_list: - jobs = glob.glob('%s*' % ejob) - for job in jobs: - move_files(job) - os.chdir(cwd) diff --git a/utils/new-publish/README b/utils/new-publish/README deleted file mode 100644 index b9be0c4..0000000 --- a/utils/new-publish/README +++ /dev/null @@ -1,111 +0,0 @@ -Prototype of new Publishing API for Linaro CI -============================================= - -Background ---------------------- -Builds of various products and components must finish with publishing -their artifacts to a central server, hereafter called "snapshots". -Builds also must be queued for testing in LAVA. All publishing -should happen in secure manner, prohibiting direct system break-ins -and minimizing types of other attacks, like denial of service. - -This prototype tries to establish consistent external interface reusable -for wild variety of Linaro builds, and initial implementation which -works with existing infrastructure and setup in place. - -Generalize publishing process: - -Builder -> Snapshots - - -External Interface ------------------- -Build jobs use publishing API using the shell command calls. To -perform publishing build calls following script: - -publish --token= --type= --strip= ... - - - Token to authenticate publishing request. It is expected that security - token is injected into build process by top-level scheduler. [Not - implemented in prototype.] - - Type of the build from a predefined set, like "android", "kernel", - "openembedded", etc. Generally, this selects target area for publishing, - but may influence other build parameters, like directory structure, - metadata, etc. - - String number of components from paths produced by . - - Build ID of the form /. This allows identification - of particular build job and its specific build case. build_id is usually - used directly as path (URL) component to access build artifacts. - - Shell glob patterns to capture artifact files. There may be more than one, - separate by spaces, or (for compatibility with Jenkins), by commas (in this - case no spaces allowed). Patterns must follow shell syntax, i.e. - multi-level match (**) is not supported. - -Example: - -$ publish --token=SECRET --type=android --strip=2 panda/10 out/target/*.tar.bz2 - -With this command, artifacts can be expected to be found on URL like - -http://snapshosts/android/panda/10/*.tar.bz2 - -Internal Implementation ------------------------ -There's currently no token-based authentication for publishing services, -and instead SSH auth used. Consequenetly, for security reasons, the accounts -used for publishing should be as restricted as possible, in practice we -use few accounts for each step of the process, each fortified to disallow -opportunity of direct shell access. SFTP is used as a transport (due to -historical reasons). - -Current publisher process goes as: - -Builder -> Master -> Snapshots - -Publishing starts on build slave with SFTPing artifact files to master -(using one account with chrooted SFTP access), then triggering further -processing by calling out (by SSH) sshd-config fixed script on master. -This script recursively applies same processing (chroot SFTP, fixed script) -to publish files to snapshots. - -Conclusions and Future Work ---------------------------- -The biggest management and security issue with the implementation described -above is authentication of publishing clients to publishing service. -Implementation described above is cumbersome to setup and maintain and -doesn't adhere to strictest security practices. - -To adress this problem, implementation of publishing as a web service may be -suggested - this way, authentication handling on server side is confined to -a single custom component, web application. It thus can be very flexible -and featureful, for example, we can implement "publishing tockens", each -associated with set of constraints, like "active not before 30min from -time of issuance", "active not after 2hr from time of issuance", "can -be used for publishing type 'android'", "publisher IP should be X.X.X.X", -etc., etc. However, there still remains problems of issuing tockens for -build hosts. Essentially, tockens should be "injected" into builds by -a trusted party (a kind of build scheduling frontend). We already have -frontend on android-build, but ci.linaro.org presents "raw" Jenkins. It -might be possible to integrate needed functionality into Jenkins via plugin. - -But publishing few moderately-sized files is not the only usecase for -Publishing Service. For OpenEmbedded builds, we need to publish used sources/ -cache files, which may be thousands of files totalling gigabytes. Except -that any particular build would like likely change only reasonably small -subset of these files, and only those need to bt actually published. -This is clearly a usecase for rsync, but with rsync, we would need to deal -with PAM for any custom authentication, and it's still unclear if it will -possible to achieve flexibility simalar to tokens described. - -That's the dichotomy we have - we need efficient transfer protocol, as -we potentially deal with many files and large amounts of data, and yet -we need flexible token/ticket style authentication. It may be possible -to choose a compromise between the two - implement a webservice with -rudimentary "file freshness" protocol (which would work on the level of -entire file, not sub-blocks). Existing system-level ticketing systems -like Kerberos can be also considered. diff --git a/utils/new-publish/clean-uploads b/utils/new-publish/clean-uploads deleted file mode 100755 index f637a70..0000000 --- a/utils/new-publish/clean-uploads +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# -# Cronjob to clean up uploads area on master -# - -find /mnt/publish/uploads/ -mindepth 1 -mmin +120 | xargs -n100 rm -rf diff --git a/utils/new-publish/propagate.py b/utils/new-publish/propagate.py deleted file mode 100755 index aa12d33..0000000 --- a/utils/new-publish/propagate.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -"""This script propagates build artifacts from build master host -to actual publishing location (snapshots).""" -import sys -import os -import optparse - -import paramiko - -import publib - - -REMOTE_HOST_PRODUCTION = "snapshots.linaro.org" -REMOTE_HOST_STAGING = "staging.snapshots.linaro.org" -PUBLISH_USER_NAME = "linaro-android-build-publish" -TRIGGER_USER_NAME = "linaro-android-build-publish-trigger" -PUBLISH_KEY_FILE = "/home/ubuntu/snapshots-sync2.new/linaro-android-build-publish" -TRIGGER_KEY_FILE = "/home/ubuntu/snapshots-sync2.new/linaro-android-build-publish-trigger" -LOCAL_UPLOAD_DIR = "/mnt/publish/uploads" -REMOTE_UPLOAD_DIR = "/uploads/android" - -def log(msg): - print msg - sys.stdout.flush() - - -if __name__ == "__main__": - optparser = optparse.OptionParser(usage="%prog ") - optparser.add_option("-s", "--staging", action="store_true", help="Publish to staging server") - optparser.add_option("--identity-publish", metavar="KEY", default=PUBLISH_KEY_FILE, help="Publish SSH key file") - optparser.add_option("--identity-trigger", metavar="KEY", default=TRIGGER_KEY_FILE, help="Trigger SSH key file") - optparser.add_option("-n", "--dry-run", action="store_true", help="Don't actually publish files, log commands") - optparser.add_option("--host", help="Override destination publishing host, for debugging") - optparser.add_option("--step", default="all", help="Run only specific step") - options, args = optparser.parse_args(sys.argv[1:]) - if len(args) != 1: - optparser.error("Wrong number of arguments") - - publib.validate_build_id(args[0]) - - print "Starting propagation phase" - - if options.staging: - remote_host = REMOTE_HOST_STAGING - opt_staging = "-s" - else: - remote_host = REMOTE_HOST_PRODUCTION - opt_staging = "" - if options.host: - remote_host = options.host - - if options.step in ("all", "1"): - file_list = [] - for root, dirs, files in os.walk(os.path.join(LOCAL_UPLOAD_DIR, args[0])): - file_list.extend([os.path.join(root, f) for f in files]) - print "Files:", file_list - strip = len(LOCAL_UPLOAD_DIR.strip("/").split("/")) - dir_list = publib.make_dir_struct(file_list, REMOTE_UPLOAD_DIR, strip=strip) - print "Dirs:", dir_list - if not options.dry_run: - log("Creating dir structure on upload server") - publib.create_dir_struct(dir_list, remote_host, PUBLISH_USER_NAME, - options.identity_publish) - log("Done creating dir structure on upload server") - upload_script = publib.make_upload_script(file_list, REMOTE_UPLOAD_DIR, strip=strip) - log("Uploading files to upload server") - publib.upload_files(upload_script, remote_host, PUBLISH_USER_NAME, - options.identity_publish, options) - log("Done uploading files to upload server") - - if options.step in ("all", "2"): - job, build = args[0].split("/") - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect(remote_host, username=TRIGGER_USER_NAME, key_filename=TRIGGER_KEY_FILE) - log("Triggering moving of files from upload to download area") - stdin, stdout, stderr = client.exec_command("reshuffle-files -t android -j %s -n %s -m %s" % (job, build, opt_staging)) - stdin.close() - rc = stdout.channel.recv_exit_status() - print "Moving phase completed with result: %d" % rc - print "=== stdout ===" - print stdout.read() - print "=== stderr ===" - print stderr.read() - print "==============" - client.close() - sys.exit(rc) diff --git a/utils/new-publish/publib.py b/utils/new-publish/publib.py deleted file mode 100755 index 9d39025..0000000 --- a/utils/new-publish/publib.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import glob -import tempfile -import optparse - -import paramiko - - -REMOTE_HOST = "android-build.linaro.org" -USER_NAME = "publish-copy" -UPLOAD_DIR = "/uploads" -KEY_FILE = "/var/run/lava/publish-copy" - -def validate_build_id(id): - if "\\" in id: - pass - elif ".." in id: - pass - elif id[0] == "/": - pass - elif len(id.split("/")) != 2: - pass - else: - return True - - print "Invalid build ID" - sys.exit(1) - - -def get_file_list(patterns): - expanded = [] - for p in patterns: - for direl in glob.glob(p): - if os.path.isfile(direl): - expanded.append(direl) - expanded.sort(lambda a, b: cmp(a.rsplit("/", 1), b.rsplit("/", 1))) - return expanded - -def common_path_prefix(s1, s2): - "Find common prefix string of s1 & s2. Return (prefix, rest_of_s1)" - s1 = s1.strip("/").split("/") - s2 = s2.strip("/").split("/") - if len(s1) > len(s2): - l = len(s2) - else: - l = len(s1) - for i in xrange(l): - if s1[i] != s2[i]: - return s1[:i], s1[i:] - return s1[:l], s1[l:] - -def strip_path_comps(path, num): - path = path.strip("/") - return "/".join(path.split("/")[num:]) - -def make_dir_struct(file_list, upload_dir, build_dir="", strip=0): - script = [] - file_list = map(lambda f: strip_path_comps(f, strip), file_list) - - file_list = map(lambda f: os.path.join(build_dir, f), file_list) - # Skip top-level files - dir for them is pre-existing - file_list = filter(lambda f: "/" in f, file_list) - dir_list = map(lambda f: os.path.dirname(f), file_list) - dir_list = list(set(dir_list)) - dir_list.sort() - prev_d = '' - for d in dir_list: - created, to_create = common_path_prefix(d, prev_d) - p = os.path.join(upload_dir, "/".join(created)) - for comp in to_create: - p = os.path.join(p, comp) - script.append(p) - prev_d = d - return script - -def create_dir_struct(dir_list, host, user, key): - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect(host, username=user, key_filename=key) - sftp = paramiko.SFTPClient.from_transport(client.get_transport()) - for d in dir_list: - try: - sftp.listdir(d) - except: - sftp.mkdir(d, 0755) - - -def make_upload_script(file_list, upload_dir, build_dir="", strip=0): - script = [] -# if dir and dir[0] != "/": -# dir = "/" + dir - last_dir = None - for f in file_list: - target_f = strip_path_comps(f, strip) - # Prepend UPLOAD_DIR now, to avoid special case of no dir name - # in file name - target_f = os.path.join(upload_dir, build_dir, target_f) - dirname, fname = target_f.rsplit("/", 1) - if dirname != last_dir: - script.append("cd %s" % dirname) - last_dir = dirname - script.append("put %s" % f) - return script - - -def upload_files(upload_script, host, user, key, options): - fd, fname = tempfile.mkstemp(prefix="sftp_script") - os.close(fd) - f = open(fname, "w") - f.write("\n".join(upload_script) + "\n") - f.close() - - cmd = "sftp -i %s -b %s %s@%s" % (key, fname, user, host) - print cmd - sys.stdout.flush() - if not options.dry_run: - rc = os.system(cmd) - try: - os.remove(fname) - except: - pass - if rc != 0: - print "ERROR: sftp transfer finished with error" - sys.exit(1) diff --git a/utils/new-publish/publish b/utils/new-publish/publish deleted file mode 100755 index b18e351..0000000 --- a/utils/new-publish/publish +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import glob -import optparse - -import publib - - -REMOTE_HOST = "android-build.linaro.org" -COPY_USER_NAME = "publish-copy" -TRIGGER_USER_NAME = "publish-trigger" -UPLOAD_DIR = "/uploads" -COPY_KEY_FILE = "/var/run/lava/publish-copy" -TRIGGER_KEY_FILE = "/var/run/lava/publish-trigger" - -def log(msg): - print msg - sys.stdout.flush() - -if __name__ == "__main__": - optparser = optparse.OptionParser(usage="%prog ...") - optparser.add_option("-t", "--type", help="Build type") - optparser.add_option("-p", "--strip", metavar="NUM", type=int, default=0, - help="Strip leading components from dest file names.") - optparser.add_option("-s", "--staging", action="store_true", help="Publish to staging") - optparser.add_option("--identity-copy", metavar="KEY", default=COPY_KEY_FILE, help="SSH key file") - optparser.add_option("--identity-trigger", metavar="KEY", default=TRIGGER_KEY_FILE, help="SSH key file") - optparser.add_option("-n", "--dry-run", action="store_true", help="Don't actually publish files, log commands") - optparser.add_option("--host", help="Override destination publishing host, for debugging") - options, args = optparser.parse_args(sys.argv[1:]) - if len(args) < 2: - optparser.error("Wrong number of arguments") - - build_id = args[0] - patterns = args[1:] - - publib.validate_build_id(build_id) - - # Support Jenkins syntax, with comma as separator - if len(patterns) == 1 and "," in patterns[0]: - patterns = patterns[0].split(",") - - for p in patterns: - if "**" in p: - print "** in glob patterns is not supported" - sys.exit(1) - - file_list = publib.get_file_list(patterns) - print "File list:", file_list - dir_list = publib.make_dir_struct(file_list, UPLOAD_DIR, build_id, options.strip) - print "Dir list:", dir_list - log("Creating dir structure on intermediate server") - publib.create_dir_struct(dir_list, REMOTE_HOST, COPY_USER_NAME, options.identity_copy) - log("Done creating dir structure on intermediate server") - upload_script = publib.make_upload_script(file_list, UPLOAD_DIR, build_id, options.strip) - log("Uploading files to intermediate server") - publib.upload_files(upload_script, REMOTE_HOST, COPY_USER_NAME, options.identity_copy, options) - log("Done uploading files to intermediate server") - - log("Triggering propagation of files to downloads server") - rc = os.system("ssh -i %s %s@%s propagate.py %s %s %s" % (options.identity_trigger, - TRIGGER_USER_NAME, REMOTE_HOST, - "-s" if options.staging else "", - "--host=%s" % options.host if options.host else "", - build_id)) - if rc != 0: - log("Publishing failed") - sys.exit(1) - - log("Publishing successful") diff --git a/utils/new-publish/setup.sh b/utils/new-publish/setup.sh deleted file mode 100755 index 2d27d27..0000000 --- a/utils/new-publish/setup.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -set -e - -publish_home=/mnt/publish - -# Append content to a file if grep test fails -function add_if_absent() { - file=$1 - exists_regexp=$2 - to_add=$3 - if grep -q -E "$exists_regexp" "$file"; then - echo "Warning $file matched $exists_regexp, adding new content skipped" - return - fi - echo "$to_add" >>$file -} - -# Comment out (#) a line if it matches regexp -function comment_if_present() { - file=$1 - regexp=$2 - awk "\ -/$regexp/ {print \"#\" \$0; next; } - { print } -" $file > $file.tmp - mv $file.tmp $file -} - -function setup_accounts() { - - mkdir -p $publish_home - - groupadd publish || true - - useradd \ - --comment "Publishing - transfer user" \ - --home $publish_home \ - --gid publish \ - -M \ - --no-user-group \ - publish-copy || true - - useradd \ - --comment "Publishing - finalization user" \ - --home $publish_home \ - --gid publish \ - -M \ - --no-user-group \ - publish-trigger || true - - # Home dir must be owned by root for ssh ChrootDirectory to work - chown root.root $publish_home - chmod 755 $publish_home - - # Actual uploads will happen to this dir - mkdir -p $publish_home/uploads - # publish-copy should have write access there, publish-trigger - # generally only read (cleanup can be handled by cronjob) - chown publish-copy.publish $publish_home/uploads - chmod 755 $publish_home/uploads -} - - -function setup_sshd_config() { - sshd_config=/etc/ssh/sshd_config -# sshd_config=sshd_config - if [ ! -f $sshd_config.before-new-pub ]; then - # Capture pristine config for rollback - cp $sshd_config $sshd_config.before-new-pub - fi - - add_if_absent $sshd_config "^AuthorizedKeysFile /etc/ssh/user-authorized-keys/%u" "\ -AuthorizedKeysFile /etc/ssh/user-authorized-keys/%u -AuthorizedKeysFile2 /dev/null" - - comment_if_present $sshd_config "^Subsystem sftp" - add_if_absent $sshd_config "^Subsystem sftp internal-sftp" "Subsystem sftp internal-sftp" - - add_if_absent $sshd_config "^Match User publish-copy" "\ -Match User publish-copy - ChrootDirectory $publish_home - ForceCommand internal-sftp - AllowTcpForwarding no - X11Forwarding no -" -} - -function setup_ssh_keys() { - mkdir -p /etc/ssh/user-authorized-keys/ - # Move only real file, don't do this for symlink - if [ ! -L $HOME/.ssh/authorized_keys ]; then - echo "Moving current account's authorized_keys to /etc/ssh/user-authorized-keys/" - cp $HOME/.ssh/authorized_keys /etc/ssh/user-authorized-keys/$SUDO_USER - rm $HOME/.ssh/authorized_keys - ln -s /etc/ssh/user-authorized-keys/$SUDO_USER $HOME/.ssh/authorized_keys - fi - # Unlike when reside in ~/.ssh/, in /etc/ssh/... auth keys must be readable enough, - # or won't be picked up by sshd => login lockout. - chmod 644 /etc/ssh/user-authorized-keys/$SUDO_USER - - mkdir -p ~/snapshots-sync3 - if [ ! -f ~/snapshots-sync3/publish-copy ]; then - ssh-keygen -t rsa -N "" -f ~/snapshots-sync3/publish-copy - fi - if [ ! -f ~/snapshots-sync3/publish-trigger ]; then - ssh-keygen -t rsa -N "" -f ~/snapshots-sync3/publish-trigger - fi - - - echo -n 'command="/usr/lib/sftp-server",no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding ' \ - >/etc/ssh/user-authorized-keys/publish-copy - pubkey=$(cat ~/snapshots-sync3/publish-copy.pub) - add_if_absent /etc/ssh/user-authorized-keys/publish-copy "^$pubkey" "$pubkey" - - echo -n 'command="/home/ubuntu/new-publish/utils/new-publish/trigger ${SSH_ORIGINAL_COMMAND#* }",no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding ' \ - >/etc/ssh/user-authorized-keys/publish-trigger - pubkey=$(cat ~/snapshots-sync3/publish-trigger.pub) - add_if_absent /etc/ssh/user-authorized-keys/publish-trigger "^$pubkey" "$pubkey" -} - -setup_accounts -# Setup new ssh keys structure first, or there's a chance of SSH lock-out -setup_ssh_keys -setup_sshd_config diff --git a/utils/new-publish/test/data/dir1/dir1_file b/utils/new-publish/test/data/dir1/dir1_file deleted file mode 100644 index e69de29..0000000 diff --git a/utils/new-publish/test/data/dir2/dir2_file b/utils/new-publish/test/data/dir2/dir2_file deleted file mode 100644 index e69de29..0000000 diff --git a/utils/new-publish/test/data/dir2/subdir1/dir2_subdir1_file b/utils/new-publish/test/data/dir2/subdir1/dir2_subdir1_file deleted file mode 100644 index e69de29..0000000 diff --git a/utils/new-publish/test/data/top_file b/utils/new-publish/test/data/top_file deleted file mode 100644 index e772b13..0000000 --- a/utils/new-publish/test/data/top_file +++ /dev/null @@ -1 +0,0 @@ -top_file \ No newline at end of file diff --git a/utils/new-publish/test/test_publish.py b/utils/new-publish/test/test_publish.py deleted file mode 100644 index 57d4c6c..0000000 --- a/utils/new-publish/test/test_publish.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -import publib - - -def data_path(): -# return os.path.join(os.path.dirname(__file__), "data") - return "test/data/" - -def test_file_list1(): - l = publib.get_file_list([data_path() + "/*"]) - assert l == ['test/data/top_file'], l - -def test_file_list2(): - l = publib.get_file_list([data_path() + "/*/*"]) - assert l == ['test/data/dir1/dir1_file', 'test/data/dir2/dir2_file'], l - -def test_file_list3(): - d = data_path() - l = publib.get_file_list([d + "/*", d + "/*/*"]) - assert l == ['test/data/top_file', 'test/data/dir1/dir1_file', 'test/data/dir2/dir2_file'], l - -def test_file_list_order(): - d = data_path() - l = publib.get_file_list([d + "*", d + "dir2/*", d + "dir1/*"]) - assert l == ['test/data/top_file', 'test/data/dir1/dir1_file', 'test/data/dir2/dir2_file'], l - -def test_common_prefix(): - v = publib.common_path_prefix("a", "b") - assert v == ([], ["a"]), v - v = publib.common_path_prefix("a", "a") - assert v == (["a"], []), v - v = publib.common_path_prefix("a/c", "a/b") - assert v == (["a"], ["c"]), v - -def test_dir_creation(): - l = publib.make_dir_struct(["file1"], "/uploads") - assert l == [] - l = publib.make_dir_struct(["file1", "dir2/f", "dir1/f"], "/uploads") - assert l == ['/uploads/dir1', '/uploads/dir2'], l - l = publib.make_dir_struct(["dir/subdir/f", "dir/f"], "/uploads") - assert l == ['/uploads/dir', '/uploads/dir/subdir'], l - l = publib.make_dir_struct(["dir/s1/s2/f", "dir/f"], "/uploads") - assert l == ['/uploads/dir', '/uploads/dir/s1', '/uploads/dir/s1/s2'], l - - -def test_ftp_script_trivial(): - l = publib.make_upload_script(["file1"], "/uploads") - assert l == ['cd /uploads', 'put file1'], l - l = publib.make_upload_script(["dir/file1"], "/uploads") - assert l == ['cd /uploads/dir', 'put dir/file1'], l - - -def test_validate_build_id(): - assert publib.validate_build_id("foo/bar") - assert publib.validate_build_id("foo/bar-2.5") - - try: - publib.validate_build_id("foo/bar/baz") - assert False - except SystemExit: - pass - - try: - publib.validate_build_id("foo/bar\\baz") - assert False - except SystemExit: - pass - - try: - publib.validate_build_id("../passwd") - assert False - except SystemExit: - pass diff --git a/utils/query-jobs/README b/utils/query-jobs/README deleted file mode 100644 index c703f94..0000000 --- a/utils/query-jobs/README +++ /dev/null @@ -1,25 +0,0 @@ -This is a script to query Jenkins job configs, on the level of raw XML files. - -Run ./query-jobs --help for list of optoins. - - -Example usage: - -./query-jobs //hudson.triggers.TimerTrigger --filter - - List all regularly repeating jobs. - -./query-jobs "disabled='false' and //hudson.triggers.TimerTrigger" --filter - - List all regularly repeating jobs, taking into account that some of them may be - disabled. - -./query-jobs //hudson.triggers.TimerTrigger/spec --filter -v - - List crontab specs for all regularly repeating jobs. - -./query-jobs //daysToKeep --filter-not --filter-jobname='-(release|toolchain)' - - List daily (not toolchain and not release) jobs which don't have build - expriation configured (i.e. would use up disk space without limit). - diff --git a/utils/query-jobs/query-jobs b/utils/query-jobs/query-jobs deleted file mode 100755 index d0b7971..0000000 --- a/utils/query-jobs/query-jobs +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -import sys -import glob -import re -import optparse - -from lxml import etree - - -JENKINS_HOME = "/var/lib/jenkins/" - -def get_xpath(el): - path = [] - while el is not None: - path.insert(0, el.tag) - el = el.getparent() - return "/" + "/".join(path) - -def match_job_name(job_name): - "Check if job name matches filters which may be specified on command line." - if not options.filter_jobname: - return True - neg = False - r = options.filter_jobname - if r[0] == "-": - neg = True - r = r[1:] - return bool(re.search(r, job_name)) ^ neg - -def process(query): - for fname in glob.glob(options.home + "jobs/*/config.xml"): - job_name = fname.split('/')[-2] - if not match_job_name(job_name): - continue - - doc = etree.parse(fname) - results = doc.getroot().xpath(query) - if options.filter and not results: - continue - if options.filter_not and results: - continue - - if options.verbose: - print "===== %s =====" % (fname if options.show_file else job_name) - else: - print fname if options.show_file else job_name - if options.verbose: - first = True - for el in results: - if not first: - print "-" * 20 - print "Element XPath: %s" % get_xpath(el) - print etree.tostring(el) - first = False - -optparser = optparse.OptionParser(usage="%prog") -optparser.add_option("-v", "--verbose", action="store_true", help="Verbose") -optparser.add_option("--show-file", action="store_true", help="Show job config path (default is job name)") -optparser.add_option("--home", default=JENKINS_HOME, help="Jenkins home dir (%default)") -optparser.add_option("--filter-jobname", metavar="REGEX", help="Process only jobs matching regex pattern (prefix with '-' for not matching)") -optparser.add_option("--filter", action="store_true", help="Show jobs matching query") -optparser.add_option("--filter-not", action="store_true", help="Show jobs not matching query") -options, args = optparser.parse_args(sys.argv[1:]) -if len(args) != 1: - optparser.error("Wrong number of arguments") - -process(args[0]) diff --git a/utils/slave-logs/archive-slave-logs b/utils/slave-logs/archive-slave-logs deleted file mode 100755 index b7a1a64..0000000 --- a/utils/slave-logs/archive-slave-logs +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -# -# This script moves EC2 build slave logs, which short-sightedly -# put straight into JENKINS_HOME by EC2 plugin, into slave-logs/ subdir. -# While moving, it also prefixes each log with a date, to facilitate -# log analysis, querying and grepping. -# -# It is intended to be run as a cronjob. -# - -JENKINS_HOME=/var/lib/jenkins - -cd $JENKINS_HOME -mkdir -p slave-logs - -find . -maxdepth 1 -type f -name 'slave-i*.log*' -mmin +60 -print0 | \ - xargs -0 --no-run-if-empty ls -l | \ - awk ' {print $8 " " "slave-logs/" $6 "-" substr($8, 3)}' | \ - xargs --no-run-if-empty -n2 mv -b diff --git a/utils/slave-logs/jenkins-clean-slave-logs b/utils/slave-logs/jenkins-clean-slave-logs deleted file mode 100755 index c207a14..0000000 --- a/utils/slave-logs/jenkins-clean-slave-logs +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# Remove old cloud build slave logs -find /var/lib/jenkins/ -maxdepth 1 -regex '.+/slave-.*\.log\(\.[0-9]+\)?' -mtime +20 | sudo xargs --no-run-if-empty rm diff --git a/utils/slave-logs/merge-logs.sh b/utils/slave-logs/merge-logs.sh deleted file mode 100755 index 125cb05..0000000 --- a/utils/slave-logs/merge-logs.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# archive-slave-logs script may move not yet complete logs, so -# later new log with the same will be created (usually it has -# just one line though: "Connection terminated"). We copy with -# mv --backup, so original file will be backed up as *~. This -# script both such parts together (again, 2nd part usually -# contains just one line, but to be on safe side and fuly correct). -# -# This script should be run with sudo -# - -cd /var/lib/jenkins/slave-logs - -for f in 20*~; do - main=`echo $f | sed -e 's/~//'` - echo "cat $main >> $f && mv $f $main" -done -- cgit v1.2.3