#!/usr/bin/python
# Author: Sandro Tosi <morph@debian.org>
# Date: 2008-10-26
# License: public domain
# 
# This script merges 2 lists of orphaned packages:
#
#  - the ones from http://qa.debian.org/data/orphaned.txt
#    (or from cli, if the file is already on the disk)
#  - the lists of package maintained by Debian QA Group
#
# then it tries to partition this set into many subsets
# in order to facilate the possible adoption by teams or
# people interested in particular packages areas

from __future__ import with_statement
import sys
import urllib
import apt_pkg
import pprint
import subprocess
import ldap, re # for wnpp bugs retrival
from mako.template import Template # for output page

def append_to_dict(dict, key, value, separator='; '):
    """Appends a value to a dict entry (if it exists) or create the entry (if not); it creates the dictionary too, if it doesn't exist"""
    if dict is None:
        dict = {}
    if key in dict:
        dict[key] = dict[key] + separator + value
    else:
        dict[key] = value

def get_wnpp_bug_via_ldap(maybe):

    LDAP_ADDR = 'bts2ldap.debian.net'
    LDAP_PORT = 10101
    BASE   = 'dc=current,dc=bugs,dc=debian,dc=org'

    try:
        __wnpp = ldap.open (LDAP_ADDR, LDAP_PORT)
        __wnpp.bind (BASE, '', ldap.AUTH_SIMPLE)
    except ldap.LDAPError, error:
        raise RuntimeError, "Cannot connect to LDAP server: %s" % error[1]
    except ldap.SERVER_DOWN:
        raise RuntimeError, "Cannot connect to LDAP server"

    ATTRS = ('debbugsID', 'debbugsTitle')

    wnpp_bugs_tmp = {}

    for mkey in maybe.keys():
        for pmkey in maybe[mkey].keys():
            SEARCH = '(&(&(debbugsTitle=*: %s --*)(debbugsPackage=wnpp))(debbugsState=open))' % pmkey
            res = __wnpp.search_s (BASE, ldap.SCOPE_BASE, SEARCH, ATTRS)
            if res:
                wnpp_bugs_tmp[pmkey] = res[0][1]
    return wnpp_bugs_tmp


orph_not_qa = []
wnpp_bugs = {}

if len(sys.argv) == 1:
    print "I: Reading orphaned.txt packages from qa.debian.org"
    orph = urllib.urlopen('http://qa.debian.org/data/orphaned.txt')
else:
    print "I: Reading orphaned.txt from first cli input parameter"
    orph = open(sys.argv[1])

# Parse orphaned.txt to obtain a list of packages
num = int(orph.readline().strip())
while orph.readline():
    p = {}
    p["package"] = orph.readline().strip()
    if not p["package"]:
        break
    p["maintainer"] = orph.readline().strip()
    p["bugid"] = int(orph.readline().strip())
    p["subject"] = orph.readline().strip()
    p["submitter"] = orph.readline().strip()
    p["age"] = int(orph.readline().strip())
    orph_not_qa.append(p["package"])
    wnpp_bugs[p["package"]] = p["bugid"]

orph.close()

# Obtain the packages already maintained by QA Group
# so that have received a QA upload
print "W: please execute '/usr/sbin/sync-available >/dev/null' before running this script"
proc = subprocess.Popen("grep-available -F Maintainer 'QA' -s Source:Package | sort -u | awk '{ print $2 }'",
                       shell=True,
                       stdout=subprocess.PIPE,
                       )
orph_qa_out = proc.communicate()[0]
# split the stdout in lines
orph_qa = orph_qa_out.split('\n')

# init apt structures
apt_pkg.init()
srcrecords = apt_pkg.GetPkgSrcRecords()
cache = apt_pkg.GetCache()
depcache = apt_pkg.GetDepCache(cache)

# init subsets (represented as dict)
#maybe_python = {}
#maybe_perl = {}
#maybe_game = {}
#maybe_lib = {}
#maybe_ruby = {}
#maybe_java = {}
#maybe_sci = {}
#maybe_kde = {}
#maybe_gnome = {}
#
uncat = {}
maybe = {'python':{},'perl':{},'games':{},'lib':{},'sci':{},'gnome':{},'kde':{},'java':{},'ruby':{},'tex':{}}

# debugging
import pdb;# pdb.set_trace()


print "I: Analyzing orphaned packages"

for pkg in orph_not_qa + orph_qa:
    try:
        # setting the current package
        srcrecords.Lookup(pkg)

        # remove 'contrib/' 'non-free/' from section
        section = srcrecords.Section.replace('contrib/','').replace('non-free/','')

        # trying to identify category from source package section
        if section == 'python':
            append_to_dict(maybe['python'], pkg, 'section = '+ srcrecords.Section)
        elif section == 'perl':
            append_to_dict(maybe['perl'], pkg, 'section = '+ srcrecords.Section)
        elif section == 'games':
            append_to_dict(maybe['games'], pkg, 'section = '+ srcrecords.Section)
        elif section in ('libs', 'oldlibs', 'libdevel'):
            append_to_dict(maybe['lib'], pkg, 'section = '+ srcrecords.Section)
        elif section in ('science', 'electronics', 'math'):
            append_to_dict(maybe['sci'], pkg, 'section = '+ srcrecords.Section)
        elif section == 'gnome':
            append_to_dict(maybe['gnome'], pkg, 'section = '+ srcrecords.Section)
        elif section == 'kde':
            append_to_dict(maybe['kde'], pkg, 'section = '+ srcrecords.Section)
        elif section == 'tex':
            append_to_dict(maybe['tex'], pkg, 'section = '+ srcrecords.Section)
        else:
            # in case of uncategorized section, we print section
            append_to_dict(uncat, srcrecords.Section, '.', separator='')
            #print srcrecords.Section
        

        # trying to identify category from source package build-depends
        for builddep in srcrecords.BuildDepends:
            if builddep[0] in ['python','python-all','python-dbg','python-dev']:
                append_to_dict(maybe['python'], pkg, 'build-depends on '+builddep[0])
            if builddep[0] in ['perl']:
                append_to_dict(maybe['perl'], pkg, 'build-depends on '+builddep[0])
            if builddep[0] in ['ruby']:
                append_to_dict(maybe['ruby'], pkg, 'build-depends on '+builddep[0])
            if builddep[0] in ['sun-java5-jdk','sun-java6-jdk','java-gcj-compat','default-jdk']:
                append_to_dict(maybe['java'], pkg, 'build-depends on '+builddep[0])

        # trying to identify category from binary packages information
        #  -> TODO <-
        #for bin in srcrecords.Binaries:
        #    binpkg = depcache.GetCandidateVer(cache[bin])

        srcrecords.Restart()
    except KeyboardInterrupt:
        print "E: Exiting, as per user request"
        exit(-1)
    except:
        #pdb.set_trace()
        print "W: Problem examining", pkg

#print "maybe: ",
#pprint.pprint(maybe)
#print
#print "uncat: ",
#pprint.pprint(uncat)

print "I: Retriving WNPP bugs via LDAP interface"
wnpp_bugs = get_wnpp_bug_via_ldap(maybe)

#import pprint
#pprint.pprint(wnpp_bugs)

print "I: Generating HTML report"

mytemplate = Template(filename='qa_packages_analyze.mako')
webpage = mytemplate.render(maybe=maybe,uncat=uncat,wnpp_bugs=wnpp_bugs)

with open('qa_packages_analyze.html','w') as f:
    f.write(webpage)

################################################################
# TODO: add pkgs already under QA team, using python-apt or debian-python?
#
# maybe python
# -> depends on python
#
# maybe perl
# -> depends on perl
#
# maybe ruby
# -> depends on ruby
#
# maybe java
# -> depends on java|j2sdk|openjdk|whatever?
#
# maybe game
# -> short or long description containing game
#
# maybe scientific
# -> short or long description containing "scien" (ce,tific) or so
#
# maybe library
# -> name lib* (ma non lib-*perl|ruby?) 
# -> section devel
#
# other maybe: sound/mail/net/graphics/web