path: root/generate-cimonitor-dashboard.py

#!/usr/bin/python3

# Usage :
#   generate-cimonitor-dashboard.py cimonitor-configs/TCWG.yaml /public_html/
#

import sys
import json
import os
import yaml
import datetime
import re
import tempfile
import traceback

scripts_dir=os.path.dirname(sys.argv[0])

import argparse
parser = argparse.ArgumentParser(description='Generate ci monitor html files',
                    prog='generate-cimonitor-dashboard.py',
                    usage='%(prog)s -o public_html [config_files]+')
parser.add_argument('-o', '--output', type=str, nargs=1,
                    help='Output directory of html generated files')
parser.add_argument('configs', type=str, nargs='+',
                    help='Yaml config files to describe html format')


########################################################################################################################
# Basic, low level functions
def nice_print(data):
	json_formatted_str = json.dumps(data, indent=4)
	print(json_formatted_str)

def download_and_open(url):
	fd, tmpnm=tempfile.mkstemp()
	os.system("wget " + "-o /dev/null " + "-O " + tmpnm + " " + url)
	return os.fdopen(fd, 'r'), tmpnm

def close_and_remove(tmpf, tmpnm):
	tmpf.close()
	os.remove(tmpnm)

dt_now=datetime.datetime.now()
def days_since(timestamp):
	return round(((int(dt_now.strftime('%s')) - round(timestamp/1000)) / 3600) / 24)
def time_since(timestamp):
	nhours=round((int(dt_now.strftime('%s')) - round(timestamp/1000)) / 3600)
	ndays=int(nhours / 24)
	nhours=round( nhours - ndays*24 )
	return "%02ddays %02dhrs ago"%(ndays, nhours)

########################################################################################################################
## CONFIG FILE
ci_url="https://ci.linaro.org/"
ci_url_view=ci_url+"view/"
ci_url_job=ci_url+"job/"
ci={}

"""
Read and load yaml config file as it is."""
def get_config(config_name):
	with open(config_name, 'r') as file:
	    config = yaml.safe_load(file)
	config_sanity_check(config)
	# nice_print(config)
	all_configs=config_instantiate_on_all_pattern(config)
	# nice_print(all_configs)
	return all_configs

def config_sanity_check(config):
	# TO BE DONE
	# nice_print(config)
	if 'format' not in config:
		assert("format not exists")

def config_instantiate_on_all_pattern(config):
	all_configs=[]

	# Assume no pattern means LNT (static) dashboard, keep as-is
	if 'pattern' not in config:
		all_configs.append(config)
		return all_configs

	all_jobs=get_ci_page("https://ci.linaro.org", request="/api/json?tree=jobs[name]")
	for pattern in config['pattern']:
		config_fmt=config['format']
		instantiated_config={}
		instantiated_config['filename']=re.sub("@pattern@", pattern, config_fmt['filename'])

		# summary_tables
		if 'summary_table_all_runs' in config_fmt:
			instantiated_config['summary_table_all_runs']=config_fmt['summary_table_all_runs']
		if 'summary_table_last_run' in config_fmt:
			instantiated_config['summary_table_last_run']=config_fmt['summary_table_last_run']

		# links
		if 'links' in config_fmt:
			instantiated_config['links']=[]
			for j in range(0, len(config_fmt['links'])):
				if re.search('@pattern@', config_fmt['links'][j]):
					# Get matching pattern on all config[pattern] list
					link_pattern=re.sub("@pattern@", pattern, config_fmt['links'][j])
					for pat in config['pattern']:
						if link_pattern==pat or not re.search(pattern, pat):
							continue

						if re.search(link_pattern+"-", pat): char="-"
						elif re.search(link_pattern+"_", pat): char="_"

						# print("ref=%s tst=%s (char=%s)" %(pattern,pat,char))
						if pat.count(char)>link_pattern.count(char)+1:
							continue
						instantiated_config['links'].append(pat)

				else:
					instantiated_config['links'].append( config_fmt['links'][j] )

		# details_table
		instantiated_config['details_table']={}
		instantiated_config['details_table']['columns']=config_fmt['details_table']['columns']
		instantiated_config['details_table']['lines']=[]
		for j in range(0, len(config_fmt['details_table']['lines'])):
			if re.search('@pattern@', config_fmt['details_table']['lines'][j]) or re.search('\*', config_fmt['details_table']['lines'][j]):
				# Get matching pattern on all CI jobs
				if pattern=="tcwg": pattern="tcwg_"
				pjt_pattern = re.sub("@pattern@", pattern, config_fmt['details_table']['lines'][j])
				for job in all_jobs['jobs']:
					if re.search("^"+pjt_pattern+"$", job['name']):
						instantiated_config['details_table']['lines'].append( job['name'] )
			else:
				instantiated_config['details_table']['lines'].append( config_fmt['details_table']['lines'][j] )

		all_configs.append(instantiated_config)

	return all_configs


########################################################################################################################
## COMPUTE MESSAGE ROUTINES
"""
Compute best message to display using internal ci-status representation
- compute_smart_status()
- compute_smart_diag()
- compute_color()
"""

########################
# compute_smart_status
"""
compute_smart_status()

Status reported can any stage of RR algorithm :
   init / success / reducing / bisecting / forced / failure
"""
def compute_smart_status(build):
	ret_attr={'text':"-", 'hlink':"", 'class':"", 'color':""}

	if not build['result']:
		return ret_attr

	# default status (Success, failure, aborted)
	ret_attr['text']=build['result']
	ret_attr['color']=compute_color(build, ret_attr['text'])

	# refine with , displayName, nb_components
	displayname=build['displayName']
	components=re.sub("^#[0-9]*(.*-)R.*",r'\1',displayname)
	nb_components=components.count("-")-1

	if re.search(r'.*-force', displayname):
		ret_attr['text']="FORCED"
	elif re.search(r'.*-init', displayname):
		ret_attr['text']="INIT"
	elif re.search(r'.*-trigger-bisect', displayname):
		ret_attr['text']="BISECTING"
	elif re.search(r'slowed down|grew in size|vect reduced|sve reduced|failed to build', displayname):
		ret_attr['text']="REGRESSED"
	elif nb_components==1 and 'actions' in build:# and 'parameters' in build['actions']:
		params=[]
		for act in build['actions']:
			if 'parameters' in act:
				params=act['parameters']
		for param in params:
			if re.sub("-(.*)-",r'\1_git', components) == param['name'] and \
			   param['value'] != "default":
				ret_attr['text']="REDUCING"
				break;
	
	build['rr_status']=ret_attr
	return ret_attr;

########################
# compute_smart_diag
"""
compute_smart_diag()

It mainly reads lastBuild project, artifacts/results, and console to compute the best diag for this build
"""
def compute_smart_diag(project, build):
	ret_attr={'text':"-", 'hlink':"", 'class':"", 'color':""}
	#ret_attr['hlink']=ci_url_job+project['project_name']+"/lastCompletedBuild"+"/artifact/artifacts/results"+"/*view*/"
	if 'result' in build and build['result'] == "SUCCESS":
		file, tmpf = download_and_open(ci_url_job+project['project_name']+"/"+str(build['number'])+"/artifact/artifacts/results-vs-prev/csv-results-1/status.csv")
		nb_failed=0
		nb_passed=0
		if not file:
			return ret_attr
		ret_attr={'text':"none", 'hlink':"", 'class':"", 'color':""}
		for items in file:
			if re.search("failed-to-build", items) or re.search("failed-to-run", items):
				nb_failed=nb_failed+1
			elif re.search("success", items):
				nb_passed=nb_passed+1
		close_and_remove(file, tmpf)
		if nb_failed!=0 or nb_passed!=0:
			ret_attr['text']="%d fails out of %d"% (nb_failed, nb_failed+nb_passed)
		return ret_attr

	# return diag if found
	file, tmpf = download_and_open(ci_url_job+project['project_name']+"/"+str(build['number'])+"/artifact/artifacts/results")
	last_step="-"
	for items in file:
		if re.search("Benchmarking infra is offline", items):
			ret_attr['text']="Board is offline"
			ret_attr['class']='diag'
			break
		elif re.search("slowed down", items):
			ret_attr['text']="slowed down"
			ret_attr['class']='diag'
			break
		elif re.search("speeds up", items):
			ret_attr['text']="speeds up"
			ret_attr['class']='diag'
			break
		elif re.search("grew in size", items):
			ret_attr['text']="grew in size"
			ret_attr['class']='diag'
			break
		elif re.search(r'vect reduced|sve reduced', items):
			ret_attr['text']="vect/sve reduced"
			ret_attr['class']='diag'
			break

		elif re.search("build errors in logs", items):
			ret_attr['text']="Build errors in : "+last_step
			break

		elif re.search("internal compiler error", items):
			ret_attr['text']="ICE in : "+last_step
			break

		elif re.search(r'^# .*(reset_artifacts|build_abe|build_bmk_llvm|benchmark|linux_n_obj)', items):
			last_step=re.sub("^# ","", items)
			last_step=re.sub(" --.*","", last_step)
			last_step=re.sub(":","", last_step)
			ret_attr['text']=last_step
	close_and_remove(file, tmpf)

	file, tmpf = download_and_open(ci_url_job+project['project_name']+"/"+str(build['number'])+"/consoleText")
	build_machine=""
	for items in file:
		if re.search("No space left on device", items):
			ret_attr['text']="No space left on device"
			ret_attr['class']='diag'
			break
		elif re.search(r'java.*Exception', items):
			ret_attr['text']="Java Exception"
			ret_attr['class']='diag'
			break
			break
		elif re.search("Build timed out", items):
			ret_attr['text']="Build timed out"
			ret_attr['class']='diag'
			break
		elif re.search("STOPPING at .* due to failure", items):
			ret_attr['text']=re.sub("STOPPING at (.*) due to failure\n",r'\1', items)
			ret_attr['class']='diag'
			break
		elif re.search("CARRYING ON after failure in .*", items):
			ret_attr['text']=re.sub("CARRYING ON after failure in (.*)\n",r'\1', items)
			ret_attr['text']=re.sub("build_abe ",'', ret_attr['text'])
			ret_attr['class']='diag'
			break
		elif re.search("^ERROR: .* failed", items):
			ret_attr['text']=re.sub("ERROR: (.*) failed\n",r'\1', items)
			ret_attr['class']='diag'
			break
	close_and_remove(file, tmpf)

	if ret_attr['color'] == "":
		ret_attr['color']=compute_color(build, ret_attr['text'])

	if ret_attr['text'] != "-":
		return ret_attr

	# Otherwise returns last stepg
	ret_attr['text']=last_step
	return ret_attr


"""
compute_color()

Choose best color
"""
def compute_color(pjt_info_build, text):
	# CI failure
	if re.search(r'ABORTED', text):
		return "purple"
	elif re.search(r'Board is offline|Java Exception|Board is offline|No space left on device', text):
		return "purple"

	# failure (normal flow)
	elif re.search(r'FAILURE', text):
		return "red"

	elif re.search(r'REGRESSED|REDUCING|BISECTING', text):
		return "#E57373" # lightred

	# Sucess (normal flow)
	elif re.search(r'FORCED|INIT|SUCCESS', text):
		return "green"

	elif not pjt_info_build or not pjt_info_build['result']:
		return ""

	elif re.search(r'ABORTED', pjt_info_build['result']):
		return "purple"
	elif re.search(r'SUCCESS', pjt_info_build['result']):
		return "green"

	return ""


"""
get_artifact_format_version()

Get version of artifact format <major>.<minor>
"""
def get_artifact_format_version(project_name):
	file, tmpf = download_and_open(ci_url_job+project_name+"/artifact/artifacts/manifest.sh")
	major="-"
	minor="-"
	for items in file:
		if re.search("rr\[major\]=", items):
			major=re.sub("rr\[major\]=\"([0-9]*)\"",r'\1', items.strip())
		elif re.search("rr\[minor\]=", items):
			minor=re.sub("rr\[minor\]=\"([0-9]*)\"",r'\1', items.strip())
	close_and_remove(file, tmpf)
	return "%s.%s" % (major,minor)

########################################################################################################################
## BUILD CI STATUS REPRESENTATION
"""
Get info from CI server and build ci-status representation
- get_ci_page()
- get_ci_project_infos()
- get_ci_project_attribute()
- get_ci_project()
- get_ci()
"""

"""
get_ci_page()

Download CI page as json.
"""
ci_pages={}
def get_ci_page(url, request=""):
	global ci_pages
	if url+request in ci_pages:
		return ci_pages[url+request]
	else:
		#print(".", end='')
		try:
			print("wget "+url)
			os.system("wget " + "-o /dev/null " + "-O /tmp/download.json " + url + "/api/json" + request)
			f = open('/tmp/download.json')
			ci_pages[url+request]=json.load(f)
			f.close()
		except:
			ci_pages[url+request]={}
		return ci_pages[url+request]


"""
get_ci_project_infos()

Retrieve the information from a given project out of the CI server
"""
def get_ci_project_infos(pjt_name):
	ci_pjt={}
	usual_requests="?tree=number,result,timestamp,displayName,building,inQueue"
	usual_requests+=",builds[number,building,result,timestamp,displayName,actions[parameters[name,value]]]"
	usual_requests+=",actions[causes[upstreamUrl,upstreamBuild],parameters[name,value]]"

	ci_pjt=get_ci_page(ci_url_job+pjt_name, request=usual_requests)
	ci_pjt['project_name']=pjt_name

	if 'builds' not in ci_pjt: return ci_pjt

	for bld in ci_pjt['builds']:
		if not bld['result']:
			continue

		if 'lastCompletedBuild' not in ci_pjt:
			ci_pjt['lastCompletedBuild']=bld
		if 'lastSuccessfulBuild' not in ci_pjt and bld['result']=='SUCCESS':
			ci_pjt['lastSuccessfulBuild']=bld
		if 'lastFailedBuild' not in ci_pjt and not bld['result']=='SUCCESS':
			ci_pjt['lastFailedBuild']=bld
		if 'lastRegressedBuild' not in ci_pjt and re.search(r'.*-trigger-bisect', bld['displayName']):
			ci_pjt['lastRegressedBuild']=bld

			bisect_name=re.sub("-build$", r'-bisect', pjt_name)
			bld2=get_ci_page(ci_url_job+bisect_name, request=usual_requests)
			# May need to check if -bisect/lastBuild bld is the upstream of -build/lastRegressedBuild
			ci_pjt['lastRegressedBuild']['bisectJob']={}
			ci_pjt['lastRegressedBuild']['bisectJob']['project_name']=bisect_name
			if 'inQueue' in bld2:
				ci_pjt['lastRegressedBuild']['bisectJob']['inQueue']=bld2['inQueue']
			if 'builds' in bld2 and 0 in bld2['builds'] and bld2['builds'][0]:
				ci_pjt['lastRegressedBuild']['bisectJob']['building']=bld2['builds'][0]['building']
				ci_pjt['lastRegressedBuild']['bisectJob']['displayName']=bld2['builds'][0]['displayName']
				ci_pjt['lastRegressedBuild']['bisectJob']['result']=bld2['builds'][0]['result']
				ci_pjt['lastRegressedBuild']['bisectJob']['timestamp']=bld2['builds'][0]['timestamp']
		if 'lastForcedBuild' not in ci_pjt and re.match('.*-force', bld['displayName']):
			ci_pjt['lastForcedBuild']=bld

			bld2=get_ci_page(ci_url_job+pjt_name+"/"+str(bld['number']), request=usual_requests)
			if not bld2: continue
			for action in bld2['actions']:
				if 'causes' in action:
					for cause in action['causes']:
						if 'upstreamUrl' in cause:
							ci_pjt['lastForcedBuild']['upstreamUrl']=cause['upstreamUrl']
							ci_pjt['lastForcedBuild']['upstreamBuild']=cause['upstreamBuild']
							bld3=get_ci_page(ci_url+cause['upstreamUrl']+str(cause['upstreamBuild']), request=usual_requests)
							if bld3:
								ci_pjt['lastForcedBuild']['upstreamBuildName']=bld3['displayName']
		compute_smart_status(bld)

	return ci_pjt


"""
get_ci_project_attribute()

Get one info from the CI server (1 project - 1 attribute)
"""
def get_ci_project_attribute(pjt_infos, attr_name):
	ret_attr={'text':"-", 'hlink':"", 'class':"", 'color':""}

	#if True:
	try:
		if attr_name=="project":
			ret_attr['text']=pjt_infos['project_name']
			ret_attr['hlink']=ci_url_job+pjt_infos['project_name']

		elif attr_name=="display_name" or attr_name=="last_title":
			ret_attr['text']=pjt_infos['lastCompletedBuild']['displayName']
			ret_attr['hlink']=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])

		elif attr_name=="build_number":
			ret_attr['text']=pjt_infos['lastCompletedBuild']['number']
			ret_attr['hlink']=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])

		elif attr_name=="pure_status":
			ret_attr=pjt_infos['lastCompletedBuild']['rr_status']

		elif attr_name=="diag":
			ret_attr=compute_smart_diag( pjt_infos, pjt_infos['lastCompletedBuild'] )

		elif attr_name=="status":
			ret_status=pjt_infos['lastCompletedBuild']['rr_status']
			ret_diag=compute_smart_diag( pjt_infos, pjt_infos['lastCompletedBuild'] )
			if ret_diag['text'] != "-":
				ret_attr['text']=ret_status['text']+" ("+ret_diag['text']+")"
			else:
				ret_attr['text']=ret_status['text']
			ret_attr['color']=compute_color(pjt_infos['lastCompletedBuild'], ret_attr['text'])
			ret_attr['text']="<input type=\"checkbox\">" + ret_attr['text']

		elif attr_name=="since_last_build":
			ret_attr['text']=time_since(pjt_infos['lastCompletedBuild']['timestamp'])

		elif attr_name=="since_last_success":
			ret_attr['text']=time_since(pjt_infos['lastSuccessfulBuild']['timestamp'])

		elif attr_name=="since_last_fail":
			ret_attr['text']=time_since(pjt_infos['lastFailedBuild']['timestamp'])

		elif attr_name=="since_last_force":
			ret_attr['text']=time_since(pjt_infos['lastForcedBuild']['timestamp'])

		elif attr_name=="since_last_regressed":
			ret_attr['text']=time_since(pjt_infos['lastRegressedBuild']['timestamp'])

		elif attr_name=="nb_force":
			total=0
			forced=0
			for bld in pjt_infos['builds']:
				total=total+1
				if re.match('.*-force', bld['displayName']):
					forced=forced+1
			ret_attr['text']="%02d%% (%d/%d)"% (int(forced/total*100), forced, total)

		elif attr_name=="nb_fail":
			total=0
			failed=0
			#nice_print(pjt_infos['builds'])
			for bld in pjt_infos['builds']:
				total=total+1
				if not bld['result']=='SUCCESS':
					failed=failed+1
			ret_attr['text']="%.2f%% (%d out of %d)"% (forced/total, forced, total)

		# Useful links
		elif attr_name=="useful_links":
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/results/*view*/"
			ret_attr['text']="<a href="+lnk+">res</a>"

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/console"
			ret_attr['text']+=" / <a href="+lnk+">console</a>"

		elif attr_name=="last_build":
			if 'lastCompletedBuild' not in pjt_infos:
				return {'text':"never", 'color':"#90A4AE", 'hlink':"", 'class':""}
			ret_attr['text']=time_since(pjt_infos['lastCompletedBuild']['timestamp'])

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])
			ret_attr['text']+=" : <a href="+lnk+">"+str(pjt_infos['lastCompletedBuild']['displayName'])+"</a>"

		elif attr_name=="last_success":
			if 'lastSuccessfulBuild' not in pjt_infos:
				return {'text':"never", 'color':"#90A4AE", 'hlink':"", 'class':""}
			ret_attr['text']=time_since(pjt_infos['lastSuccessfulBuild']['timestamp'])

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastSuccessfulBuild']['number'])
			ret_attr['text']+=" : <a href="+lnk+">"+str(pjt_infos['lastSuccessfulBuild']['displayName'])+"</a>"

		elif attr_name=="last_fail":
			if 'lastFailedBuild' not in pjt_infos:
				return {'text':"never", 'color':"#90A4AE", 'hlink':"", 'class':""}
			ret_attr['text']=time_since(pjt_infos['lastFailedBuild']['timestamp'])

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastFailedBuild']['number'])
			ret_attr['text']+=" : <a href="+lnk+">"+str(pjt_infos['lastFailedBuild']['displayName'])+"</a>"

		elif attr_name=="last_forced":
			if 'lastForcedBuild' not in pjt_infos:
				return {'text':"never", 'color':"#90A4AE", 'hlink':"", 'class':""}
			ret_attr['text']=time_since(pjt_infos['lastForcedBuild']['timestamp'])

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastForcedBuild']['number'])
			ret_attr['text']+=" : <a href="+lnk+">"+str(pjt_infos['lastForcedBuild']['displayName'])+"</a>"

			lnk=ci_url+pjt_infos['lastForcedBuild']['upstreamUrl']+str(pjt_infos['lastForcedBuild']['upstreamBuild'])
			ret_attr['text']+=" (<a href="+lnk+">bisect</a> : "

			bldname=pjt_infos['lastForcedBuild']['upstreamBuildName']
			if re.match(r'.*spurious|.*baseline', bldname):
				bldname=re.sub("#([0-9]*)-(.*)-(.*)", r'\2-\3', bldname)
				lnk=False
			else:
				compon=re.sub("#([0-9]*)-(.*)-(.*)", r'\2', bldname)
				sha1=re.sub("#([0-9]*)-(.*)-(.*)", r'\3', bldname)
				bldname = compon+"-"+sha1[0:7]
				if not re.match('tcwg_bmk.*speed', pjt_infos['project_name']):
					lnk="https://git.linaro.org/toolchain/ci/interesting-commits.git/tree/%s/sha1/%s"%(compon, sha1)

			if lnk:
				ret_attr['text']+="<a href="+lnk+">"+bldname+"</a>)"
			else:
				ret_attr['text']+=bldname+")"

		elif attr_name=="last_regressed":
			if 'lastRegressedBuild' not in pjt_infos:
				return {'text':"never", 'color':"#90A4AE", 'hlink':"", 'class':""}
			ret_attr['text']=time_since(pjt_infos['lastRegressedBuild']['timestamp'])

			total=0
			matched=0
			for bld in pjt_infos['builds']:
				total=total+1
				if re.match('.*-trigger-bisect', bld['displayName']):
					matched=matched+1
			ret_attr['text']+="(%d/%d) : "% (matched, total)

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastRegressedBuild']['number'])
			ret_attr['text']+="<a href="+lnk+">"+str(pjt_infos['lastRegressedBuild']['displayName'])+"</a>"

		elif attr_name=="last_bisect":
			if 'lastRegressedBuild' not in pjt_infos or 'bisectJob' not in pjt_infos['lastRegressedBuild'] or \
			   not pjt_infos['lastRegressedBuild']['bisectJob']:
				return {'text':"never", 'color':"#90A4AE", 'hlink':"", 'class':""}
			bisect_job=pjt_infos['lastRegressedBuild']['bisectJob']
			ret_attr['text']=time_since(bisect_job['timestamp'])

			lnk=ci_url_job+bisect_job['project_name']
			state=""
			#nice_print(pjt_infos['lastRegressedBuild'])
			if 'inQueue' in bisect_job and bisect_job['inQueue']:
				state="(waiting)"
			elif 'building' in bisect_job and bisect_job['building']:
				lnk+="/lastBuild"
				state="(running)"
			elif not bisect_job['result']=="SUCCESS":
				lnk+="/lastBuild"
				state="(failed)"
			elif re.match('.*-spurious|.*-advance-baseline', bisect_job['displayName']):
				lnk+="/lastBuild"
				state="(spurious)"
			elif re.match('#[0-9]*-(.*)-[a-z0-9]{40}$', bisect_job['displayName']):
				lnk+="/lastBuild"
				state="(ok)"
			else:
				lnk+="/lastBuild"
				#print(bisect_job)
				state="("+bisect_job['displayName']+")"
			ret_attr['text']+="<a href="+lnk+">bisect"+state+"</a> / "

			ret_attr['text']+="<a href="+lnk+">"+str(bisect_job['displayName'])+"</a>"

		elif attr_name=="statistics":
			(tot, fail, forc, trig, pas)=(0, 0, 0, 0, 0)
			#nice_print(pjt_infos['builds'])
			for bld in pjt_infos['builds']:
				tot=tot+1
				if re.match('.*-force', bld['displayName']):
					forc=forc+1
				elif re.match('.*-trigger-bisect', bld['displayName']):
					trig=trig+1
				elif bld['result']=='FAILURE' or bld['result']=='ABORTED':
					fail=fail+1
				elif bld['result']=='SUCCESS':
					pas=pas+1

			ret_attr['text']="regressed(%02d%%) force(%02d%%) fail(%02d%%) ok(%02d%%) nbruns=%d"\
			  %(int(100*trig/tot), int(100*forc/tot), int(100*fail/tot), int(100*pas/tot), tot)

		elif attr_name=="artifact_version":
			if 'lastSuccessfulBuild' not in pjt_infos:
				return {'text':"--", 'color':"#90A4AE", 'hlink':"", 'class':""}

			# get_artifact_version
			ret_attr['text']=get_artifact_format_version(pjt_infos['project_name']+"/"+str(pjt_infos['lastSuccessfulBuild']['number']))

			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastSuccessfulBuild']['number'])+"/artifact/artifacts/99-rewrite/more"
			res=os.system("wget -o /dev/null -O /tmp/more "+lnk)
			if res == 0:
			    ret_attr['text']+=" (not-completed)"

		elif attr_name=="result_file":
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/results/*view*/"
			ret_attr['text']="<a href="+lnk+">res</a>"
		elif attr_name=="console":
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/console"
			ret_attr['text']="<a href="+lnk+">console</a>"

		# TODO
		elif attr_name=="notify_verif":
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/"
			ret_attr['text']="<a href="+lnk+">artifacts</a>/"
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/results/*view*/"
			ret_attr['text']+="<a href="+lnk+">results</a>/ "
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/jenkins/mail-body.txt/*view*/"
			ret_attr['text']+="<a href="+lnk+">email</a>/"
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/jenkins/jira-body.txt/*view*/"
			ret_attr['text']+="<a href="+lnk+">jira</a>/ "
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/jenkins/notify.log/*view*/"
			ret_attr['text']+="<a href="+lnk+">log</a>/"
			lnk=ci_url_job+pjt_infos['project_name']+"/"+str(pjt_infos['lastCompletedBuild']['number'])+"/artifact/artifacts/results-vs-prev/csv-results-1/status.csv/*view*/"
			ret_attr['text']+="<a href="+lnk+">status</a>"

		elif attr_name=="bmk_job":
			file, tmpf = download_and_open(ci_url_job+pjt_infos['project_name']+"/lastCompletedBuild"+"/artifact/artifacts/results_id")
			for items in file:
				ret_attr['text']=re.sub(".*/","", items)
				ret_attr['hlink']=ci_url_job+"tcwg-benchmark"+"/"+ret_attr['text']
			close_and_remove(file, tmpf)
			
		elif attr_name=="components":
			ret_attr['text']=re.sub("^#[0-9]*-(.*)-R.*",r'\1',pjt_infos['lastCompletedBuild']['displayName'])

		elif attr_name=="failing_step":
			ret_attr=compute_smart_diag(pjt_infos)

	except:
		ret_attr['text']="-"
		traceback.print_exc()

	return ret_attr

"""
get_ci_project()

Get ci-status internal representation for a project (PJT)
Iterates over the CONFIG columns, and request the infos for each one
"""
def get_ci_project(config, ci, pjt_name):
	ci_project={}
	ci_project['data']=get_ci_project_infos(pjt_name)
	for attr in config['details_table']['columns']:
		ci_project[attr] = get_ci_project_attribute(ci_project['data'], attr)
	return ci_project


def count_jobs(ci, requested_status, summary_style, mindays, maxdays):
	nb=0
	for job in ci:
		if not 'data' in ci[job] or not 'lastCompletedBuild' in ci[job]['data']:
			continue

		if summary_style=='summary_table_last_run':
			bld=ci[job]['data']['lastCompletedBuild']
			if 'timestamp' not in bld or not bld['rr_status']['text']:
				continue
			days=days_since(bld['timestamp'])
			if days>=mindays and days<=maxdays and re.search(requested_status, bld['rr_status']['text']):
				nb=nb+1

		elif summary_style=='summary_table_all_runs':
			for bld in ci[job]['data']['builds']:
				#nice_print(bld)
				if 'timestamp' not in bld:
					continue
				if 'rr_status' not in bld or not bld['rr_status']['text']:
					continue
				days=days_since(bld['timestamp'])
				if days>=mindays and days<=maxdays and re.search(requested_status, bld['rr_status']['text']):
					nb=nb+1

	#if nb==0: return "-"
	#else: return nb
	return nb

"""
get_ci_summary_laps()
"""
def get_ci_summary_laps(ci, timelaps, nb_of, summary_style):
	ret_attr={'text':"-", 'hlink':"", 'class':"", 'color':""}

	mindays=0
	maxdays=9999
	if re.search(r'([0-9]*)-days-ago$', timelaps):
		mindays=maxdays=int(re.sub("([0-9]*)-days-ago$",r'\1', timelaps))
	if re.search(r'last-([0-9]*)-days', timelaps):
		maxdays=int(re.sub("last-([0-9]*)-days",r'\1', timelaps))
	if re.search(r'last-week', timelaps):
		maxdays=7

	#try:
	if nb_of=="last_run_date":
			ret_attr['text']=timelaps
			ret_attr['color']="blue"
	elif nb_of[0]=="#":
			ret_attr['text']=count_jobs(ci, nb_of[1:].upper(), summary_style, mindays, maxdays)
			ret_attr['color']=compute_color(None, nb_of[1:].upper())
	#except:
	#	ret_attr['text']="-"

	return ret_attr
	

"""
get_ci()

Get ci-status internal representation main routine.
Iterates over the CONFIG project, and request the infos for each one
"""
def get_ci(config):
	ci={}
	# LNT dashboard does not have a details_table
	if 'details_table' not in config:
		return ci

	for line in config['details_table']['lines']:
		all_jobs=get_ci_page("https://ci.linaro.org", request="/api/json?tree=jobs[name]")
		for job in all_jobs['jobs']:
			if re.search(line, job['name']):
				ci[job['name']]=get_ci_project(config, ci, job['name'])
		
	if 'summary_table_last_run' in config:
		summary={}
		for timelaps in config['summary_table_last_run']['lines']:
			summary[timelaps]={}
			for nb_of in config['summary_table_last_run']['columns']:
				summary[timelaps][nb_of]=get_ci_summary_laps(ci, timelaps, nb_of, 'summary_table_last_run')
		ci['summary_table_last_run']=summary

	if 'summary_table_all_runs' in config:
		summary={}
		for timelaps in config['summary_table_all_runs']['lines']:
			summary[timelaps]={}
			for nb_of in config['summary_table_all_runs']['columns']:
				summary[timelaps][nb_of]=get_ci_summary_laps(ci, timelaps, nb_of, 'summary_table_all_runs')
		ci['summary_table_all_runs']=summary

	return ci




########################################################################################################################
## DUMP HTML

"""
dump html routines
- dump_html_one_line
- dump_html
"""

html_header = """<html>
<style>
  table, td, th {
    border-collapse: collapse;
  }
  tbody tr:nth-child(even) td {
    background-color: #ededed;
  }
</style>
<head>
<title>CI Infrastructure Status - %s</title>
<link rel="stylesheet" type="text/css" href="sorting-table-css/example.css" />
</head>
<body>
<h1>CI Infrastructure Status - %s</h1>
"""


html_footer = """
 <script>
   var table = document.querySelector('.massive')
   var tbody = table.tBodies[0]
   var rows = [].slice.call(tbody.rows, 0)
   var fragment = document.createDocumentFragment()

   for (var k = 0; k < 50; k++) {
     for (var i = 0; i < rows.length; i++) {
       fragment.appendChild(rows[i].cloneNode(true))
     }
   }
   tbody.innerHTML = ''
   tbody.appendChild(fragment)
 </script>
 <!-- <script type="text/javascript" src="sortable.js"></script> -->
 <script src="sorting-table-css/sortable.js"></script>
 <script>
   function prepareAdvancedTable() {
     function convertSizeToBytes(str) {
       var matches = str.match(/^([0-9.]+)(\w+)$/)
       if (matches) {
         var vals = {
           kB: 1, // 1024 B
           KiB: 1,// 1024 B
           MB: 2, // 1024 * 1024 B
           GB: 3, // 1024 * 1024 * 1024 B
           TB: 4, // 1024 * 1024 * 1024 *1024 B
         }
         return (matches[1] || 0) * Math.pow(1024, vals[matches[2]])
       }
       return str
     }

     var size_table = document.querySelector('.advanced-table')
     var rows = size_table.tBodies[0].rows
     for (let i = 0; i < rows.length; i++) {
       const date_element = rows[i].cells[2]
       const size_element = rows[i].cells[1]
       date_element.setAttribute('data-sort', date_element.innerText.replace(/(\d+)\/(\d+)\/(\d+)/, '$3$1$2'))
       size_element.setAttribute('data-sort', convertSizeToBytes(size_element.innerText))
     }
   }
   prepareAdvancedTable()
 </script>
</body>
</html>
"""

def dump_html_one_line(f, config_table, ci_pjt):
	f.write(" <tr>\n")
	for attr in config_table['columns']:
		f.write("  <td>")
		if ci_pjt[attr]['color']:
			f.write("<font color=\""+ci_pjt[attr]['color']+"\">")
		if ci_pjt[attr]['hlink']:
			f.write("<a href='"+ci_pjt[attr]['hlink']+"'>")
		f.write(str(ci_pjt[attr]['text']))
		if ci_pjt[attr]['hlink']:
			f.write("</a>")
		if ci_pjt[attr]['color']:
			f.write("</font>")
		f.write("</td>\n");
	f.write(" </tr>\n")

def dump_html_util(config_name, output_dirname):
	print("# Copy yaml : "+ config_name)
	os.system("cp " + config_name + " " + output_dirname)

	print("# Copy html : help.html")
	os.system("cp "+scripts_dir+"/cimonitor-configs/help.html " + output_dirname)

	print("# Copy css  : sorting-table-css")
	os.system("cp -ar "+scripts_dir+"/cimonitor-configs/sorting-table-css " + output_dirname)

def dump_html(config, ci):
	print("# Emit html : "+ config['filename'])
	os.system("mkdir -p "+os.path.dirname(output_dirname+"/"+config['filename']))
	f = open(output_dirname+"/"+config['filename'], 'w')

	f.write(html_header % (os.path.basename(config['filename']), os.path.basename(config['filename'])))
	f.write("<p> date = "+str(datetime.datetime.now())+"</p>\n")
	f.write("<p> config = <a href="+os.path.basename(config_name)+">"+os.path.basename(config_name)+"</a></p>\n")

        # LNT dashboard (upstream projects health)
	if 'lnt_dashboard' in config:
		f.write("<h2>LNT dashboard (upstream projects health)</h2>\n")
		f.write("<ul>\n")
		for project in config['lnt_dashboard']:
			f.write("  <h1>"+project+"</h1>\n")
			for entry in config['lnt_dashboard'][project]:
				text=entry.split()
				f.write("  <li><a href=\""+text[1]+"\">"+text[0]+"</a></li>\n")
		f.write("</ul>\n\n")

	# LINKS
	if 'links' in config:
		f.write("<h2> LINKS </h2>\n")
		f.write("<ul>\n")
		for lnk in config['links']:
			f.write("  <li><a href=\""+lnk+".html\">"+lnk+".html</a></li>\n")
		f.write("</ul>\n\n")

	# SUMMARY_TABLE (Last run)
	if 'summary_table_last_run' in config:
		f.write("<h2> SUMMARY_TABLE (Last completed build only)</h2>\n")
		f.write("<p> statistics on last-completed build of all jenkins job</p>\n")
		f.write("<table border=1 cellspacing=1 cellpadding=3 class=\"sortable\">\n")
		f.write(" <thead>\n")
		for col in config['summary_table_last_run']['columns']:
			f.write("  <th>"+col+"</th>\n")
		f.write(" </thead>\n")
		for laps in config['summary_table_last_run']['lines']:
			dump_html_one_line(f, config['summary_table_last_run'], ci['summary_table_last_run'][laps])
		f.write("</table>\n\n")

	# SUMMARY_TABLE (All runs)
	if 'summary_table_all_runs' in config:
		f.write("<h2> SUMMARY_TABLE (All completed runs)</h2>\n")
		f.write("<p> statistics on all completed builds of all jenkins job</p>\n")
		f.write("<table border=1 cellspacing=1 cellpadding=3 class=\"sortable\">\n")
		f.write(" <thead>\n")
		for col in config['summary_table_all_runs']['columns']:
			f.write("  <th>"+col+"</th>\n")
		f.write(" </thead>\n")
		for laps in config['summary_table_all_runs']['lines']:
			dump_html_one_line(f, config['summary_table_all_runs'], ci['summary_table_all_runs'][laps])
		f.write("</table>\n\n")

	# DETAILS_TABLE
	if 'details_table' in config:
		f.write("<h2> DETAILS_TABLE </h2>\n")
		f.write("<p> help for the table format : <a href=\"help.html\">here</a>")
		f.write("<table border=1 cellspacing=1 cellpadding=3 class=\"sortable\">\n")
		f.write(" <thead>\n")
		for col in config['details_table']['columns']:
			f.write("  <th>"+col+"</th>\n")
		f.write(" </thead>\n")

		for pjt in config['details_table']['lines']:
			dump_html_one_line(f, config['details_table'], ci[pjt])

		f.write("</table>\n")

	f.write("<p> time to build html = "+str(datetime.datetime.now() - dt_now)+"</p>\n")

	f.write(html_footer)
	f.close()

########################################################################################################################
## BASIC ASCII DUMP ROUTINES

"""
Basic ascii dump routines
- dump_ascii_one_line
- dump_ascii
"""
def dump_ascii_fmt(k):
	if k=="project":
		return "%70s,"
	else:
		return "%20s,"

def dump_ascii_one_line(config_table, ci, pjt_name):
	for attr in config_table['columns']:
		fmt=dump_ascii_fmt(attr)
		print(fmt % (ci[pjt_name][attr]['text']), end='')
	print("")


def dump_ascii(config, ci):
	i=0
	print("\n\n=== DETAILS TABLE - %s" %(config['filename']))

	# Array header
	for col in config['details_table']['columns']:
		fmt=dump_ascii_fmt(col)
		print(fmt % (col), end='');
	print("")

	for pjt in config['details_table']['lines']:
		dump_ascii_one_line(config['details_table'], ci, pjt)


########################################################################################################################
## MAIN PROCEDURE

if __name__ == "__main__":
	args = parser.parse_args(sys.argv)

	output_dirname=args.output[0]
	if not os.path.isdir(output_dirname):
		parser.print_help()
		exit(1)

	for config_name in args.configs[1:]:

		all_configs=get_config(config_name)

		for config in all_configs:
			ci=get_ci(config)
			dump_html(config, ci)
			# dump_ascii(config, ci)

		dump_html_util(config_name, output_dirname)
	print("# Time to generated all html files : " + str(datetime.datetime.now() - dt_now))