path: root/rngen.py

#!/usr/bin/env python
# vim: set tw=78 expandtab tabstop=4 shiftwidth=4 autoindent smartindent:

import csv
import sys, getopt
reload(sys)
sys.setdefaultencoding('utf8')

# For fileIO
import os.path

# Generate html files from textile files.
import textile

# Generate text files from html.
#from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup

# Jinja2 is the templating engine.
from jinja2 import Template, FileSystemLoader, Environment

from custom_wordwrap import do_customwordwrap

from cStringIO import StringIO

from datetime import datetime

from shutil import copyfile

# To test if a remote file exists.
import urllib2

from series import Series
from linaroseries import LinaroSeries
from rnseries import CandidateRN
from gccclone import GCCClone

from rninput import finput

def get_gcc_version(gccclone):
    dct= {}
    (dct["major"],dct["minor"],dct["revision"]) = gccclone.get_base_version().split(".")
    return dct 

def rngen(candidate, gccclone):
    #TODO Test that candidate is a CandidateRN

    trackseries=candidate.get_track_series()
    nextseries=candidate.get_next_series()

    print "toNext: " + nextseries.getBranchname()
    print "toNext: " + format(nextseries, '%N/%d')

    # TODO: What if 'track' is not a snapshot branch?  To do this right we'd
    # probably have to preserve basedon information in the release notes
    # repository as a yaml file.
    basedon='http://snapshots.linaro.org/components/toolchain/gcc-linaro/'
    basedon=basedon + trackseries.getDir()
    print "basedon: " + basedon
    basis_url=basedon

#    print 'gccbaseversion is ' + gccclone.get_base_version()
#    dct= {}
#    (dct["major"],dct["minor"],dct["revision"]) = gccclone.get_base_version().split(".")
#    print 'fsf revision is ' + gccclone.get_fsf_revision()
    ver=get_gcc_version(gccclone)
    print 'gcc major ' + ver['major']
    print 'gcc minor ' + ver['minor']

    fsf=gccclone.get_fsf_revision()

    # The version numbering scheme changed with GCC 5.  Now, every new release
    # has the major version incremented.  Prior to version five the minor was
    # incremented per-release.
    # TODO: read stable and maintenance information from a config file.
    if int(ver['major']) <=  4:
        gcc_version_path= u'' + ver['major'] + '.' + ver['minor']
        stab_maint='Maintenance'
    else:
        gcc_version_path=ver['major']
        stab_maint='Stable'

    print "This is a %s candidate for GCC version %s" % (stab_maint,
            gcc_version_path)

    history=finput('Please enter the location of the changelog csv file: ', "6.1-2016.06.csv")

    print "using the following .csv file: " + history

    #month_year=format(nextseries, '%D')
    month_year=nextseries.date

    outfile=u'GCC-' + ver['major'] + u'.' + ver['minor'] + u'-' + month_year.strftime("%Y.%m")

#    if spin != "":
#        outfile=outfile + u'-' + spin
#
#    if rc != "":
#        outfile=outfile + u'-rc' + rc
    outfile=outfile + str(nextseries.spin) + str(nextseries.rc)
    print "outfile: " + outfile

    with open(history, mode='r') as backports:
        backports.seek(0)
        backports_content = backports.read()
        in_memory_backports = StringIO(backports_content)

        backportsDict = csv.DictReader(in_memory_backports, fieldnames=['rev', 'fsf', 'linaro', 'summary', 'target', 'type', 'notes', 'backport', 'owner', 'author'], delimiter=',')
        backports.close();

    release_type=nextseries.longuppertype()
    
    spin=str(nextseries.spin).strip('-')
    rc=str(nextseries.rc).strip("-rc")

    template_dir=candidate.rntemplate.workdir()
    series_dir=candidate.rnworkdir.workdir()

    # Verify that the basis url is actually valid, otherwise the release notes
    # will contain erroneous information.
    url=urllib2
    try:
        ret = urllib2.urlopen(basis_url)
    except url.URLError as e:
        print "%s does not exist!" % basis_url
        exit(1)

    # TODO verify that the basis_url is a valid url and warn if not.
    basis=u''
    basis_ver=basis_url.strip("/")
    basis_ver=basis_ver.rsplit("/",1)[1]
    if basis_url.find("releases") == 1:
        basis="Linaro Release GCC " + basis_ver
    elif basis_url.find("-rc") == 1:
        basis="Linaro Release-Candidate GCC " + basis_ver
    else:
        basis="Linaro Snapshot GCC " + basis_ver

    release_string="Linaro GCC %s.%s-%s.%s %s" % (ver['major'],ver['minor'],month_year.strftime("%Y"), month_year.strftime("%m"),release_type)
    
    print "The release notes for this %s will be generated based on: %s" % (release_string, basis)

    # Get ready to do the rendering of the jinja2 templates into .textile
    # format.

    # relative path to the README.textile templates.
    src_readme_path=u'/components/toolchain/gcc-linaro/5/README.textile'
    bin_readme_path=u'/components/toolchain/binaries/README.textile'
 
    # absolute path to README.textile templates.
    template_src_readme=template_dir + src_readme_path
    template_bin_readme=template_dir + bin_readme_path

    # relative path to the README.textile.series templates.
    # Note, the src templates don't have an extends (.series) file.
    src_extends_path=u'/components/toolchain/gcc-linaro/5/README.textile'
    bin_extends_path=u'/components/toolchain/binaries/README.textile.series'

    # absolute path to README.textile.series template overrides.  These are
    # what are passed to the jinja2 rendering engine because they might
    # override the default templates with version specific information.
    series_src_extends=series_dir + src_extends_path
    series_bin_extends=series_dir + bin_extends_path

    # These are the README.textile files that exist in the series directory.
    series_src_readme=series_dir + src_readme_path
    series_bin_readme=series_dir + bin_readme_path

    # Copy the template README.textile over the series README.textile file
    # because the jinja2 engine will need access to the parent template.  This
    # will be overwritten with the generated README.textile later.
    copyfile(template_src_readme, series_src_readme)
    copyfile(template_bin_readme, series_bin_readme)

    # After the script runs the *_series_template_path files are overwritten
    # with the generated README.textile files.
    src_readme=series_src_readme
    bin_readme=series_bin_readme


    # Setting lstrip_blocks=True lets us use indenting in jinja2 templates. 
    env = Environment(loader=FileSystemLoader(series_dir), lstrip_blocks=True, trim_blocks=True)

    # We need to use a custom wordwrap filter that doesn't line-break urls.
    env.filters['customwordwrap'] = do_customwordwrap

    source_archive_template = env.get_template(src_extends_path)
    binary_archive_template = env.get_template(bin_extends_path)
    announce_template = env.get_template(bin_extends_path)

    # Generate the toolchain GCC source archive release notes.
    source_textile_rendering = source_archive_template.render(
        backports=backportsDict,
        GCC_base_version=ver['major'] + u'.' + ver['minor'] + u'.' + ver['revision'],
        FSF_rev=fsf,
        basis=basis,
        basis_url=basis_url,
        year=month_year.strftime("%Y"),
        dd_month=month_year.strftime("%m"),
        rc=rc,
        spin=spin,
        stab_maint=stab_maint,
        release_type=release_type,
        package="GCC",
        major=ver['major'],
        minor=ver['minor'])

    source_textile_rendering = source_textile_rendering.encode('utf-8')

    out_source=u'' + outfile
    out_binary=u'Linaro-' + outfile

    try:
        with open(out_source + u'.textile','w') as f:
            f.write(source_textile_rendering)
            print 'Wrote textile rendering to file %s' % os.getcwd() + u'/' + out_source + u'.textile'
    except IOError as e:
        sys.exit('Error rendering textile source archive release notes.')

    source_html_rendering = textile.textile(source_textile_rendering)

    try:
        with open(out_source + u'.html','w') as f:
            f.write(source_html_rendering)
            print 'Wrote html rendering to file %s' % os.getcwd() + u'/' + out_source + u'.html'
    except IOError as e:
        sys.exit('Error rendering html release notes.')

    # Generate the binary toolchain archive release notes.
    binary_textile_rendering = binary_archive_template.render(
        announce='',
        year=month_year.strftime("%Y"),
        dd_month=month_year.strftime("%m"),
        rc=rc,
        spin=spin,
        major=ver['major'],
        minor=ver['minor'],
        stab_maint=stab_maint,
        snap_rc_release=release_type)
  
    binary_textile_rendering = binary_textile_rendering.encode('utf-8')
    try:
        with open(out_binary + u'.textile','w') as f:
            f.write(binary_textile_rendering)
            print 'Wrote textile rendering to file %s' % os.getcwd() + u'/' + out_binary + u'.textile'
    except IOError as e:
        sys.exit('Error rendering textile binary archive release notes.')
  
    binary_html_rendering = textile.textile(binary_textile_rendering)
  
    try:
        with open(out_binary + u'.html','w') as f:
            f.write(binary_html_rendering)
            print 'Wrote html rendering to file %s' % os.getcwd() + u'/' + out_binary + u'.html'
    except IOError as e:
        sys.exit('Error rendering html release notes.')
 
    # Generate the binary toolchain announce message.
    announce_rendering = announce_template.render(
        announce='yes',
        year=month_year.strftime("%Y"),
        dd_month=month_year.strftime("%m"),
        rc=rc,
        spin=spin,
        major=ver['major'],
        minor=ver['minor'],
        stab_maint=stab_maint,
        snap_rc_release=release_type)

    announce_rendering = announce_rendering.encode('utf-8')
    announce_file=u'announce-' + out_binary + u'.txt'
    try:
        with open(announce_file,'w') as f:
            f.write(announce_rendering)
            print 'Wrote text rendering to file %s' % os.getcwd() + u'/' + announce_file
    except IOError as e:
        sys.exit('Error rendering text announce message.')


    # Copy the generated .textile file over the
    # readme_series_dir/README.textile template file.
    # WARNING: This must be done AFTER you generate the announce message
    # because it uses the same README.textile template as the textile rendered
    # release notes.
    copyfile(out_binary + u'.textile' ,bin_readme)
    print 'Wrote generated file ' + out_binary + u'.textile' + " to " + bin_readme

    # Copy the generated .textile file over the
    # readme_series_dir/README.textile template file.
    copyfile(out_source + u'.textile' ,src_readme )
    print 'Wrote generated file ' + out_source + u'.textile' + " to " + src_readme

    candidate.rnworkdir.add(bin_readme)
    candidate.rnworkdir.add(src_readme)
    candidate.rnworkdir.commit("Generated Release Notes for " + nextseries.label())
    candidate.rnworkdir.log(1)

    # TODO: Ask if the user would like to push the release notes changes.

    quit(0);

if __name__ == "__main__":
       main(sys.argv[1:])