path: root/bin/ldtstool

#!/usr/bin/env python3

'''
ldtstool - Online and offline parsing of ticket data (in JSON)

Note: The file format used by consumed and output by this tool is based
      on the file format zendesk provides using its "Export as JSON" tool

Prerequisites (ubuntu:16.04):

 * sudo apt -y install python3 python3-pip
 * sudo apt -y install python3-iso8601 python3-keyring
 * sudo pip3 install zdesk

If you are running an older version of ubuntu (such as 14.04) then some
of the distro provided packages may be out of date.

Generate a weekly report (the dates automatically select the previous week even
if you leave your report until mid-way through the following week):

    ldtstool fetch --since 'next-friday -13 days' \
		    --until 'next-friday -1 week' | \
    ldtstool filter --assignee daniel.thompson@linaro.org | \
    ldtstool weekly

Producing a customized HTML report (by dynamically generating markdown):

    ldtstool fetch --since 2016-06-17 > tickets.json
    ldtstool tags < tickets.json
    ldtstool filter --tags ti --strict < tickets.json | \
      ldtstool format --template '* [{id}]({quicklink}): {subject} ({status})' | \
      cmark > ti_report.html

Figuring out the number of unique users:

    ldtstool filter --tags socionext --strict < tickets.json | \
      ldtstool format --template '{requestor-email}' | \
      sort | \
      uniq

'''

import argparse
import collections
import iso8601
import json
import os
import re
import textwrap
import sys

# This code requires PYTHONPATH to include .../toys/lib/python
import toys.chart as chart
import toys.collect as collect
import toys.config as config
import toys.date as date

# If it's installed we'd rather use IPython for interaction...
try:
	import IPython
	interact = IPython.embed
except:
	import pdb
	interact = pdb.set_trace

# Try to import zdesk, failure is non-fatal since this is only needed for
# online support (the fetch sub-tool)
try:
	import zdesk
except:
	print("WARNING: zdesk module not found - no online support", file=sys.stderr)

class Ticket(dict):
	def _safe_get_string_field(self, attr, field):
		if not self[attr]:
			return ''
		val = self[attr][field]
		if not val:
			return ''
		return val

	def assignee(self, field):
		return self._safe_get_string_field('assignee', field)

	def category(self):
		hacks = {
			'11.12_android': 'android',
			'64-bit': 'kernel',
			'android_lmg': 'android',
			'big.little': 'kernel',
			'board_support': 'board',
			'eglibc' : 'toolchain',
			'home_lhg': 'home',
			'kernel_kwg': 'kernel',
			'linaro_lsk': 'kernel',
			'lava_infrastrucure': 'lava_and_qa',
			'lava': 'lava_and_qa',
			'qa': 'lava_and_qa',
			'toolchain_tcwg': 'toolchain',
			'qemu_vritualization': 'qemu',
			'qemu_virtualization': 'qemu',
			'networking_lng': 'networking',
                        'debian_ubuntu' : 'distros',
			'open_embedded_oe': 'distros',
			'open_embedded': 'distros',
			'ubuntu': 'distros',
			'power_management_pmwg': 'power',
			'power_management': 'power',
			'eas': 'power',
			'security_swg': 'security',
                        'linaro_enterprise_group_leg': 'enterprise'
		}

		# This is an ordered list of categories into which to file
		# this ticket. It will be assigned the first category found.
		globs = (
			'android', 'board', 'kernel', 'power',
		        'graphics', 'lava_and_qa', 'qemu', 'toolchain',
			'security', 'distros', '96boards', 'home',
			'networking', 'enterprise' )

		tags = [ hacks[x] if x in hacks else x for x in self['tags'] ]
		for g in globs:
			if g in tags:
				return g

		# This is a lists of tags that do *not* categorise a ticket.
		# It is used only to issue warnings (so we can detect any
		# new categories that may be created in the future and issue
		# a warning).
		nonglobs = set((
			'socionext', 'spam', 'arm', 'closed_by_merge',
			'community', 'eng_escalate', 'hisi', 'huawei',
			'internal_linaro', 'launchpad', 'level_0', 'level_1',
			'level_2', 'level_3', 'lge', 'linaro', 'open', 'other',
			'probe', 'questions_issues_on_releases', 'release',
			'releases.linaro.org', 'microsoft', 'segment_group',
			'spreadtrum', 'support', 'team', 'ti', 'advantech',
			'broadcom', 'contact_us', 'defect_created',
			'development_platforms', 'embedded_applications',
			'general_support', 'getting_involved', 'hxt', 'gts', 'juno',
			'landing_team', 'level_4', 'linaro_enterprise_group',
			'membership', 'new', 'page', 'panda',
			'qa_lava_test_scripts', 'qualcomm', 'qualcomm_stp',
			'sample', 'samsung', 'sharp_zarus_sl5500',
			'staging_vexpress_a9', 'v_fav', 'vicky_test',
			'video_condecs', 'zd_test', 'zendesk', 'arrow',
                        'variscite', 'landing_team_lt', 'code_aurora',
                        'developer_services_contract', 'baslerweb'))

		unknown = set(tags) - nonglobs
		if len(unknown):
			print("WARNING: {} has unexpected tags {}".format(
				self.id(), sorted(unknown)), file=sys.stderr)
		elif False: # Make True for verbose diagnostics
			print("WARNING: {} cannot be categorized using {}".format(
				self.id(), self['tags']), file=sys.stderr)

		return "unknown"	

	def created_at(self):
		return iso8601.parse_date(self['created_at'])

	def format(self, template):
		output = template
		for m in re.finditer('{([^}:]+)([^}]*)}', template):
			field = m.group(1)
			fmt = m.group(2)

			try:
				if 'id' == field:
					val = self.id()
				elif 'orgname' == field:
					val = self.orgname()
				elif 'quicklink' == field:
					val = self.quicklink()
				elif '-' in field:
					(field, attr) = field.split('-', 1)
					if isinstance(self[field], str):
						val = self[field][0:int(attr)]
					else:
						val = self._safe_get_string_field(field, attr)
				else:
					val = self[field]
			except ValueError:
				continue

			output = output.replace(m.group(0), '{{{}}}'.format(fmt).format(val))
		return output

	def id(self):
		return 'LDTS-{:04d}'.format(self['id'])

	def organization(self, field):
		if 'organization' not in self:
			return ''
		try:
			return self['organization'][field]
		except TypeError:
			# Occasionally the organization is an integer
			# (and therefore not sub-scriptable)
			return ''

	def orgname(self, reduce_namespace=False):
		org = self.organization('name')
		if not org or org == 'Community using gmail':
			return 'Community'

		if reduce_namespace and 'qualcomm_stp' in self['tags']:
			return 'Qualcomm'

		if org == "Segment_Group":
			email = self['requester']['email']
			org = email.split('@', maxsplit=1)[1]
			if org.count('.') == 1 and org.endswith('.com'):
				org = org[:-4].capitalize()

			if reduce_namespace:
				org = "Group member"

		return org

	def quicklink(self):
		return 'https://linaro.zendesk.com/agent/tickets/{}'.format(
				self['id'])

	def requester(self, field):
		return self._safe_get_string_field('requester', field)

	def updated_at(self):
		return iso8601.parse_date(self['updated_at'])

	def is_community(self):
		return self.orgname() == 'Community'

	def is_within_date(self, since=None, until=None, restrict=''):
		created_at = self.created_at()
		updated_at = self.updated_at()

		if 'created' in restrict:
			updated_at = created_at
		if 'updated' in restrict:
			created_at = updated_at

		if not since and not until:
			return True

		if not until:
			return created_at >= since or updated_at >= since

		if not since:
			return created_at <= until or updated_at <= until

		return (created_at >= since and created_at <= until) or \
		       (updated_at >= since and updated_at <= until)

	def is_tagged(self, needle, exact=False):
		for tag in self['tags']:
			if (exact and needle == tag) or \
			   (not exact and needle in tag):
				return True
		return False

	@staticmethod
	def fetch(since=None, until=None):
		organizations = {}
		def get_organization(id):
			if not id:
				return None

			if id not in organizations:
				organizations[id] = zd.organization_show(id)

			return organizations[id]['organization']

		users = {}
		def get_user(id):
			if not id:
				return None

			if id not in users:
				users[id] = zd.user_show(id)

			return users[id]['user']

		cfg = config.get_config()
		password = config.get_password(cfg, 'zendesk')
		zd = zdesk.Zendesk(cfg['zendesk']['server'],
				cfg['zendesk']['username'], password)

		tickets = [ Ticket(t) for t in zd.tickets_list(get_all_pages=True)['tickets'] ]

		# Filter by date as early as possible so we can reduce network round trips
		tickets = [ t for t in tickets if t.is_within_date(since, until) ]


		for t in tickets:
			t['organization'] = get_organization(t['organization_id'])
			t['assignee'] = get_user(t['assignee_id'])
			t['requester'] = get_user(t['requester_id'])

		return tickets

	@staticmethod
	def load(obj):
		if not obj:
			data = json.load(sys.stdin)
		elif isinstance(obj, str):
			with open(obj, 'r') as f:
				data = json.load(f)
		else:
			data = json.load(obj)

		return [ Ticket(t) for t in data ]

def get_count_by_member(tickets, reduce_namespace=False):
	return collect.accumulate(tickets,
			lambda t: t.orgname(reduce_namespace=reduce_namespace))

def get_count_by_month(tickets):
	count = collections.defaultdict(int)
	for t in tickets:
		# Filter out tickets there were, in all probability, spam
		if not t['assignee']:
			continue

		created = iso8601.parse_date(t['created_at'])
		count['{:04d}-{:02d}'.format(created.year, created.month)] += 1
	return count

def get_count_by_category(tickets):
	return collect.accumulate(tickets, lambda t: t.category())

def do_chart(args):
	tickets = Ticket.load(args.json)

	if args.by_assignee:
		seive = lambda t: t.assignee('name')
	elif args.by_category:
		seive = lambda t: t.category()
	else:
		seive = lambda t: t.orgname(reduce_namespace=True)

	data = collect.collate(tickets, lambda t: t.created_at().strftime('%Y-%m'))
	for month in data.keys():
		data[month] = collect.accumulate(data[month], seive)

	chart.stacked_barchart(data, args.output,
			title = 'Tickets by month and member',
			ylabel = 'Number of tickets')

def do_count(args):
	tickets = Ticket.load(args.json)
	count = None

	if args.by_member:
		count = get_count_by_member(tickets)

	if args.by_month:
		count = get_count_by_month(tickets)

	if args.by_category:
		count = get_count_by_category(tickets)

	if args.by_assignee:
		count = collect.accumulate(tickets, lambda t: t.assignee('name'))

	if count:
		flat_count = [ (k, v) for k, v in count.items() ]

		if args.rank:
			flat_count = sorted(flat_count, key=lambda x: x[1], reverse=True)
		else:
			flat_count = sorted(flat_count, key=lambda x: x[0])

		if args.csv:
			for k, v in flat_count:
				print('"{}",{}'.format(k, v))
		elif args.text:
			for k, v in flat_count:
				print('{:30} {}'.format(k, v))
		else:
			if args.rank:
				count = flat_count
			json.dump(count, sys.stdout, sort_keys=True, indent=2)
	else:
		print(len(tickets))

def do_days(args):
	for t in Ticket.load(args.json):
		created_at = iso8601.parse_date(t['created_at'])
		updated_at = iso8601.parse_date(t['updated_at'])
		days = updated_at - created_at

		print('"LDTS-{:04d}",{}'.format(t['id'], days.days+1))

def do_dump(args):
	with open(args.db) as f:
		sys.stdout.write(f.read())

def do_fetch(args):
	since = date.smart_parse(args.since)
	until = date.smart_parse(args.until)

	tickets = Ticket.fetch(since, until)

	json.dump(tickets, sys.stdout)

def do_fields(args):
	fields = set()

	for t in Ticket.load(args.json):
		for k in t.keys():
			fields.add(k)
			if isinstance(t[k], dict):
				for kk in t[k].keys():
					fields.add('{}-{}'.format(k, kk))

	for f in sorted(fields):
		print(f)

def do_filter(args):
	since = date.smart_parse(args.since)
	until = date.smart_parse(args.until)

	data = Ticket.load(args.json)

	data = [ t for t in data if t.is_within_date(since, until, args.restrict) ]

	if args.updated_before:
		before = date.smart_parse(args.updated_before)
		data = [ t for t in data if t.updated_at() < before ]

	if args.assignee:
		data = [t for t in data
			if args.assignee in t.assignee('name') or \
			   args.assignee in t.assignee('email')]
	if args.community:
		data = [ t for t in data if t.is_community() ]
	if args.member:
		data = [ t for t in data if not t.is_community() ]
	if args.tags:
		data = [ t for t in data if t.is_tagged(args.tags, args.strict) ]

	json.dump(data, sys.stdout)

def do_format(args):
	for t in Ticket.load(args.json):
		print(t.format(args.template))

def do_interact(args):
	'''Directly interact with the JSON data (or zendesk)'''
	tickets = Ticket.load(args.json)
	ticket = tickets[-1]

	try:
		cfg = config.get_config()
		zd = zdesk.Zendesk(cfg['zendesk']['server'],
				   cfg['zendesk']['username'],
				   config.get_password(cfg, 'zendesk'))
	except:
		zd = None

	print('Locals include: tickets, ticket, zd')
	interact()

def do_import(args):
	with open(args.json, 'r') as f:
		tickets = [ json.loads(ln) for ln in f.readlines() ]

	with open(args.db, 'w') as f:
		json.dump(tickets, f)

def do_markdown(args):
	template = args.template
	if args.add_organization:
		template = template + ' ({orgname})'
	if args.add_requester:
		template = template + ' ({requester-name})'
	if args.add_requester_email:
		template = template + ' ({requester-email})'

	tickets = Ticket.load(args.json)
	data = collect.collate(tickets, lambda t: t['status'])
	for status in ('closed', 'solved', 'pending', 'open', 'new'):
		if status not in data:
			continue

		print("* {}".format(status.title()))
		for t in data[status]:
			print("  - {}".format(t.format(template)))

def do_monthly(args):
	tickets = Ticket.load(args.json)

	def factory():
		return collections.defaultdict(list)
	by_member = collections.defaultdict(factory)
	for t in tickets:
		by_member[t.orgname()][t['status']].append(t)

	print("<table border=2>")
	print("<tr>")
	print("<td>Member</td>")
	print("<td>Tickets</td>")
	print("</tr>")

	def show_list(title, l):
		print("<li>{}".format(title.capitalize()))
		print("<ul>")
		for t in l:
			print("<li>")
			print('{} (<a href="{}">{}</a>)'.format(
				t['subject'], t.quicklink(), t.id()))
			print("</li>")
		print("</ul>")
		print("</li>")

	for member in sorted(by_member.keys()):
		print("<tr>")
		print("<td>{}</td>".format(member))
		print("<td><ul>")

		# Special case the community tickets
		if member == "Community":
			closed = len(by_member[member]['closed'])
			solved = len(by_member[member]['solved'])
			pending = len(by_member[member]['pending'])
			nopen = len(by_member[member]['open'])

			print("<li>{} closed/solved tickets</li>".format(
					closed+solved))
			if pending:
				print("<li>{} pending tickets</li>".format(
						pending))
			print("<li>{} open tickets</li>".format(nopen))
			print("</ul></td>")
			continue

		for status in ('closed', 'solved', 'pending', 'open'):
			if status not in by_member[member]:
				continue
			show_list(status, by_member[member][status])
			del by_member[member][status]
		for status in sorted(by_member[member].keys()):
			show_list(status, by_member[member][status])
		print("</ul></td>")
		print("</tr>")
	print("</table>")

def do_passwd(args):
	print("WARNING: This is *not* your normal password. You have to create")
	print("         an API token (and use your.email@linaro.org/token as")
	print("         your username)")
	cfg = config.get_config()
	config.set_password(cfg, 'zendesk')

def do_piechart(args):
	tickets = Ticket.load(args.json)
	count = None

	if args.by_member:
		count = get_count_by_member(tickets, reduce_namespace=True)
	else:
		count = get_count_by_category(tickets)

	chart.piechart(count, args.output, title=args.title)

def do_pull(args):
	if args.verbose:
		sys.stdout.write('Reading existing ticket cache .')
		sys.stdout.flush()
	tickets = Ticket.load(args.db)
	if args.verbose:
		sys.stdout.write('..')
		sys.stdout.flush()

	# Find the most recent update and prepare tickets for easy update
	since = tickets[0].updated_at()
	db = {}
	for t in tickets:
		d = t.updated_at()
		if d > since:
			since = d
		db[t['id']] = t

	if args.verbose:
		print(' ok')
		sys.stdout.write('Fetching tickets changed since {} ...'.format(
				since.isoformat()))
		sys.stdout.flush()

	new_tickets = Ticket.fetch(since, None)
	if args.verbose:
		print(' ok')

	for t in new_tickets:
		if t['id'] not in db:
			if args.verbose:
				print('{}: Received new ticket'.format(t.id()))
			db[t['id']] = t
			continue

		old = db[t['id']]
		if old.updated_at() < t.updated_at():
			if args.verbose:
				print('{}: Updating ticket'.format(t.id()))
			db[t['id']] = t
			continue

		if args.verbose:
			print('{}: No change'.format(t.id()))

	# Reconstruct tickets and save it
	tickets = [ db[k] for k in sorted(db.keys()) ]

	if args.pipe:
		json.dump(tickets, sys.stdout)

	with open(args.db, 'w') as f:
		json.dump(tickets, f)

def do_tags(args):
	tickets = Ticket.load(args.json)

	tags = set()
	for t in tickets:
		tags |= set(t['tags'])
	
	for t in sorted(tags):
		print(t)

def do_weekly(args):
	wrappers = (
		textwrap.TextWrapper(),
		textwrap.TextWrapper(initial_indent=' * ', subsequent_indent='   '),
		textwrap.TextWrapper(initial_indent='   - ', subsequent_indent='     ')
	)

	def wrap(msg, level=1):
		print('\n'.join(wrappers[level].wrap(msg)))

	headings = []
	for t in Ticket.load(args.json):
		organization = t.orgname()
		subject = t['subject']
		assignee = t.assignee('name').split(' ', 1)[0]
		ticket_num = t['id']
		
		headings.append('{}: {} [{}] (LDTS-{})'.format(organization, subject,
				assignee, ticket_num))
	
	for h in sorted(headings):
		wrap(h)

def do_worklog(args):
	items = []
	for t in Ticket.load(args.json):
		items.append(t.format(' * {orgname}: {subject} [{assignee-name}] ({id})'))

	if not args.time_spent:
		print("No time spent estimate. Please use the following as a memory jogger.\n")
		for i in items:
			print(textwrap.fill(i, subsequent_indent='   '))
		return

	jira = config.connect_to_jira()
	if not args.issue:
		args.issue = config.get_config()['zendesk']['jiralink']
	if args.date:
		args.date = date.smart_parse(args.date)
	jira.add_worklog(args.issue, timeSpent=args.time_spent, started=args.date, comment='\n'.join(items))

def main(argv):
	defaultdb=os.path.dirname(os.path.realpath(sys.argv[0])) + '/../zendesk.db'

	parser = argparse.ArgumentParser()
	subparsers = parser.add_subparsers(dest='sub-command')
	subparsers.required = True	# Can't be set using named arguments (yet)

	s = subparsers.add_parser('chart')
	s.add_argument('--by-assignee', action='store_true')
	s.add_argument('--by-category', action='store_true')
	s.add_argument("--output", default="ldtstool.png")
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_chart)

	s = subparsers.add_parser('count')
	s.add_argument('--by-assignee', action='store_true')
	s.add_argument('--by-category', action='store_true')
	s.add_argument('--by-member', action='store_true')
	s.add_argument('--by-month', action='store_true')
	s.add_argument('--csv', action='store_true')
	s.add_argument('--rank', action='store_true')
	s.add_argument('--text', action='store_true')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_count)

	s = subparsers.add_parser('csv')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_format,
			template='"{id}","{subject}","{orgname}","{requester-email}"')

	s = subparsers.add_parser('days')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_days)

	s = subparsers.add_parser('dump')
	s.add_argument('--db', default=defaultdb,
		       help="File to update")
	s.set_defaults(func=do_dump)

	s = subparsers.add_parser('fetch')
	s.add_argument("--since", default="2012-01-01",
			help="When to gather information from")
	s.add_argument("--until", default="tomorrow",
			help="When to stop gathering information")
	s.set_defaults(func=do_fetch)

	s = subparsers.add_parser('fields')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_fields)

	s = subparsers.add_parser("filter")
	s.add_argument("--since", default="2012-01-01",
			help="When to gather information from")
	s.add_argument("--until", default="tomorrow",
			help="When to stop gathering information")
	s.add_argument("--assignee")
	s.add_argument("--community", action='store_true')
	s.add_argument("--member", action='store_true')
	s.add_argument("--strict", action='store_true')
	s.add_argument("--restrict", default="",
		       help="'created' or 'updated' will restrict date ranges")
	s.add_argument("--tags")
	s.add_argument("--updated-before", default=None,
		       help="Mostly only useful for testing 'pull'")
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_filter)

	s = subparsers.add_parser("format")
	s.add_argument("--template",
			default="{id}: {subject} ({orgname} - {requester-email})")
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_format)

	s = subparsers.add_parser("import")
	s.add_argument('--db', default=defaultdb,
		       help="File to store imported data into")
	s.add_argument("json")
	s.set_defaults(func=do_import)

	s = subparsers.add_parser("interact")
	s.add_argument("json", nargs='?', default=defaultdb)
	s.set_defaults(func=do_interact)

	s = subparsers.add_parser('markdown')
	s.add_argument("--add-organization", action='store_true')
	s.add_argument("--add-requester", action ='store_true')
	s.add_argument("--add-requester-email", action ='store_true')
	s.add_argument("--template",
			default="[{id}]({quicklink}): {subject}")
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_markdown)

	s = subparsers.add_parser('monthly')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_monthly)

	s = subparsers.add_parser('passwd')
	s.set_defaults(func=do_passwd)

	s = subparsers.add_parser('piechart')
	s.add_argument('--by-member', action='store_true')
	s.add_argument('--by-category', action='store_true')
	s.add_argument("--output", default="piechart.png",
			help="Filename for the graph")
	s.add_argument("--title", default=None,
			help="Title of the graph")
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_piechart)

	s = subparsers.add_parser('pull')
	s.add_argument('--db', default=defaultdb,
		       help="File to update")
	s.add_argument('--pipe', action='store_true',
		       help="Duplicate output on stdout")
	s.add_argument('--verbose', action='store_true',
		       help="Show internal workings")
	s.set_defaults(func=do_pull)

	s = subparsers.add_parser('summary')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_format,
			template="{created_at-10}: {id}: {subject} ({orgname} - {requester-email})")

	s = subparsers.add_parser('tags')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_tags)

	s = subparsers.add_parser('weekly')
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_weekly)

	s = subparsers.add_parser('worklog')
	s.add_argument("--issue")
	s.add_argument("--time-spent")
	s.add_argument("--date")
	s.add_argument("json", nargs='?')
	s.set_defaults(func=do_worklog)

	args = parser.parse_args(argv[1:])
	args.func(args)

if __name__ == "__main__":
	try:
		sys.exit(main(sys.argv))
	except KeyboardInterrupt:
		sys.exit(1)
	sys.exit(127)