1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
#!/usr/bin/python
from optparse import OptionParser
import json
import logging
import os
import re
import sys
import urllib2
log = logging.getLogger("jsonmerge")
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)
def download_custom_json(url, auth=None):
request = urllib2.Request(url)
if auth:
request.add_header('Authorization', auth)
try:
response = urllib2.urlopen(request)
except urllib2.URLError, e:
if hasattr(e, 'reason'):
log.error('Failed to reach %s.' % url)
log.error('Reason: ', e.reason)
elif hasattr(e, 'code'):
log.error('ci.linaro.org could not fulfill the request: %s' % \
url)
log.error('Error code: ', e.code)
sys.exit('Failed to get last successful artifact.')
return response.read()
def substitute_template_variables(template, environ):
""" Env variable names for substitution
have to start with SUB_
Template variables are marked with
%%VARIABLE%%
Variable names need to be uppercase
"""
sub_expr = re.compile("%%(?P<variable>[_A-Z]+)%%")
for var in sub_expr.finditer(template):
var_name = var.group('variable')
log.debug("Substituting: SUB_%s" % (var_name))
var_value = environ.get("SUB_%s" % var_name)
if var_value:
template = template.replace(var.group(0), var_value)
else:
log.error("Variable SUB_%s is not set" % (var_name))
sys.exit("Failed to create valid JSON")
return template
def main():
usage = "usage: %prog [OPTIONS]"
parser = OptionParser(usage=usage)
parser.add_option("-e", "--environment", dest="environ",
help="Environment variable for running script")
parser.add_option("-m", "--metadata", dest="metadata",
help="Metadata in JSON format \
to be placed in the output definition")
parser.add_option("-i", "--input", dest="input_file",
help="Input template JSON file")
parser.add_option("-o", "--output", dest="output",
help="Output file to store JSON")
(options, args) = parser.parse_args()
if options.environ:
f = open(options.environ, "r")
for line in f.readlines():
split_result = line.split("=", 1)
if len(split_result) > 1:
os.environ[split_result[0]] = split_result[1].strip()
f.close()
if not options.input_file:
parser.error("Unable to construct job definition \
without the template")
config_tmpl = open(options.input_file, "r")
config = substitute_template_variables(config_tmpl.read(), os.environ)
config_tmpl.close()
json_config = json.loads(config)
metadata_json = None
if options.metadata:
f = open(options.metadata, "r")
metadata = f.read()
try:
metadata_json = json.loads(metadata)
except:
log.error("Unable to parse metadata file")
metadata_url = os.environ.get("METADATA_URL", None)
if metadata_url:
metadata = download_custom_json(metadata_url)
try:
metadata_json = json.loads(metadata)
except:
log.error("Unable to parse metadata URL")
if metadata_json:
if 'metadata' in json_config['actions'][0]:
json_config['actions'][0]['metadata'].update(metadata_json)
else:
json_config['actions'][0]['metadata'] = metadata_json
if options.output:
f = open(options.output, "w")
f.write(json.dumps(json_config, indent=2))
f.close()
else:
print json.dumps(json_config, indent=2)
if __name__ == '__main__':
main()
|
