path: root/mangle-jobs/mangle-jobs

#!/usr/bin/python
"""Helper to mass-edit jobs in jenkins.

"""

###############################################################################
# Copyright (c) 2011 Linaro
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
###############################################################################

from __future__ import print_function
import base64
from contextlib import nested
import json
import os
import sys
import copy
import re
from tempfile import NamedTemporaryFile
import urllib2
import optparse
import getpass
import glob
from xml.dom import minidom

from lxml.etree import fromstring, tostring


optparser = optparse.OptionParser(usage="%prog <mangle script>")
optparser.add_option("--url", default="http://localhost:8080/jenkins/",
                     help="Process jobs on Jenkins server via REST API, default: %default")
optparser.add_option("--file",
                     help="Process single local file")
optparser.add_option("--dir",
                     help="Process jobs in local Jenkins home dir")
optparser.add_option("--user",
                     help="Jenkins username (required for --url)")
optparser.add_option("--passwd-file", metavar="FILE",
                     help="File holding Jenkins password (prompt otherwise)")
optparser.add_option("--filter-jobname",
                     help="Process only jobs matching regex pattern")
optparser.add_option("--limit", type="int", default=-1,
                     help="Process at most LIMIT jobs")
optparser.add_option("--backup", action="store_true",
                     help="Backup changed files")
optparser.add_option("--really", action="store_true",
                     help="Actually perform changes, default is dry run with diff display")

options, args = optparser.parse_args(sys.argv[1:])
if len(args) != 1:
    optparser.error("Wrong number of arguments")

d = {}
execfile(args[0], d, d)
mangler = d['mangle']

password = None
if options.passwd_file:
    password = open(options.passwd_file).read().strip()
elif not options.file and not options.dir:
    password = getpass.getpass("Password/API Token:")

if options.url[-1] != '/':
    options.url += '/'

auth_headers = {
    'Authorization': 'Basic %s' % (
        base64.encodestring('%s:%s' % (options.user, password))[:-1],),
    }

def _authJenkins(jenkins_path, data=None, extra_headers=None):
    """Make an authenticated request to jenkins.

    @param jenkins_path: The path on the Jenkins instance to make the request
        to.
    @param data: Data to include in the request (if this is not None the
        request will be a POST).
    @param extra_headers: A dictionary of extra headers that will passed in
        addition to Authorization.
    @raises urllib2.HTTPError: If the response is not a HTTP 200.
    @returns: the body of the response.
    """
    headers = auth_headers.copy()
    if extra_headers:
        headers.update(extra_headers)
    req = urllib2.Request(
        options.url + jenkins_path, data, headers)
    resp = urllib2.urlopen(req)
    return resp.read()

def getJobConfig(job_name):
    return _authJenkins('job/' + job_name + '/config.xml')

def postConfig(url, configXml, extra_headers=None):
    headers = {'Content-Type': 'text/xml', }
    if extra_headers is not None:
        headers.update(extra_headers)
    _authJenkins(url, configXml, headers)

def render_xml(tree):
    # Render XML to exact dialect used by Jenkins
    # This involves some dirty magic
    text = tostring(tree, xml_declaration=True, encoding='UTF-8')
    # Roundtrip via minidom, this takes care of encoding " as entities
    tree2 = minidom.parseString(text)
    text = tree2.toxml('UTF-8')

    # expand empty tags
    text = re.sub(r"<([-A-Za-z.]+)/>", "<\\1></\\1>", text)

    # Some CR noise should be entities
    text = text.replace("\r", "&#xd;")

    # Finally, munge xml decl
    line1, rest = text.split("><", 1)
    line1 = line1.replace('"', "'")
    r = line1 + ">\n<" + rest

    return r

def show_diff(old, new, old_name="a", new_name="b"):
    with nested(NamedTemporaryFile(), NamedTemporaryFile()) as (a, b):
        a.write(old)
        b.write(new)
        a.flush(); b.flush()
        os.system('diff --label=%s --label=%s -u %s %s' % (old_name, new_name, a.name, b.name))
    print()

def indent_tree(elem, level=0):
    "Indent XML tree for pretty-printing"
    i = "\n" + level*"  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            indent_tree(elem, level+1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def normalize2text(tree):
    """Return normalized text representation of XML tree, suitable for
    diffing with normal diff tool."""
    normalized = copy.deepcopy(tree)
    indent_tree(normalized)
    return tostring(normalized)

def match_job_name(job_name):
    "Check if job name matches filters which may be specified on command line."
    if not options.filter_jobname:
        return True
    neg = False
    r = options.filter_jobname
    if r[0] == "-":
        neg = True
        r = r[1:]
    return bool(re.search(r, job_name)) ^ neg

def get_csrf_token():
    try:
        crumb_data = _authJenkins('crumbIssuer/api/xml')
    except urllib2.HTTPError:
        # Ignore errors for android-build which provides no crumb.
        return None
    tree = minidom.parseString(crumb_data)
    crumb_tag = tree.getElementsByTagName('crumb')[0]
    field_tag = tree.getElementsByTagName('crumbRequestField')[0]
    crumb = str(crumb_tag.firstChild.wholeText)
    field = str(field_tag.firstChild.wholeText)
    return (field, crumb)

def process_remote_jenkins():
    jobs = json.load(urllib2.urlopen(options.url + 'api/json?tree=jobs[name]'))
    names = [job['name'] for job in jobs['jobs']]
    names = [name for name in names if name == 'blank' or '_' in name]
    limit = options.limit

    csrf_token = get_csrf_token()
    if csrf_token is None:
        extra_headers = None
    else:
        extra_headers = { csrf_token[0]: csrf_token[1], }

    for name in names:
        if not match_job_name(name):
            continue
        if limit == 0:
            break
        limit -= 1
        print("Processing:" + name)
        sys.stdout.flush()
        org_text = getJobConfig(name)
        tree = fromstring(org_text)
        org_normalized = normalize2text(tree)

        if mangler(tree) == False:
            continue

        if not options.really:
            new_normalized = normalize2text(tree)
            show_diff(org_normalized, new_normalized)
        else:
            new_text = render_xml(tree)
            if type(new_text) == type(u""):
                new_text = new_text.encode("utf8")
            postConfig(str('job/' + name + '/config.xml'), new_text,
                       extra_headers)


def process_file(name):
    text = open(name).read()
    tree = fromstring(text)

    org_normalized = normalize2text(tree)
    if mangler(tree) == False:
        print("No changes made to %s" % name)
        sys.stdout.flush()
        return

    if not options.really:
        new_normalized = normalize2text(tree)
        show_diff(org_normalized, new_normalized, old_name=name, new_name=name + ".new")
    else:
        new_text = render_xml(tree)
        if type(new_text) == type(u""):
            new_text = new_text.encode("utf8")
        if options.backup:
            os.rename(name, name + "~")
        with open(name, "w") as f:
            f.write(new_text)


def process_dir(jenkins_home):
    limit = options.limit
    for fname in glob.iglob(jenkins_home + "/jobs/*/config.xml"):
        if limit == 0:
            break
        limit -= 1
        process_file(fname)


def main():
    if options.file:
        process_file(options.file)
    elif options.dir:
        process_dir(options.dir)
    else:
        process_remote_jenkins()

if __name__ == "__main__":
    main()