#! /usr/bin/env python3
#
# dgs.py -- generation script for the GNOME in Debian status page
# Copyright (C) 2008-2025 Frederic Peters
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import functools
import hashlib
import urllib.parse
import urllib.request
import re
import os
import sys
import apt_pkg
import subprocess
import time
import socket
import pickle
import json
from optparse import OptionParser
import io
import ssl
import tarfile

context = ssl._create_unverified_context()

STABLE = 'stable'
TESTING = 'testing'
UNSTABLE = 'unstable'

try:
    import ZSI.client
except ImportError:
    ZSI = None

try:
    import SOAPpy
except ImportError:
    SOAPpy = None

apt_pkg.init()

CACHE_DIR = 'debian-gnome-status-cache'

HTML_AT_TOP = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Status of GNOME %(version)s in Debian</title>
<script type="text/javascript" src="debian-gnome.js"></script>
<style type="text/css">
html, body {
    font-family: sans-serif;
    margin: 0;
    padding: 0;
    background: #eee;
}
#logo {
    position: absolute;
    top: 10px;
    left: 10px;
    border: 0px;
    z-index: 10;
}

#header h1 {
    margin: 0;
    padding: 5px 0px 5px 80px;
    background: #555753;
    color: white;
    margin-bottom: 1em;
    border-bottom: 2px solid #aaa;
}

div#content {
    margin: 2em;
    margin-right: 15em;
}

tfoot th, thead th {
    font-weight: normal;
}
tr.missing td, span.missing { background: #aaa; }
tr.notinsid td, span.ok-experimental { background: #fce94f; }
tr.minor td, span.lagging-minor { background: #fcaf3e; }
tr.major td, span.lagging-major { background: #ef2929; }
tr.ok td, span.ok { background: #8ae234; }
tr.ok-stable td, span.ok-stable { background: #438a00; }
tr.ok-testing td, span.ok-testing { background: #57b300; }
tr.waiting-in-new, span.waiting-in-new { background: #FCF9AF; }
tr.waiting-in-new td:last-child { display: none;}

td.heading {
    text-align: center;
    background: #555753;
    color: white;
    font-weight: bold;
}
tbody th, tbody td.pts {
    background: #d3d7cf;
    text-align: left;
}
tbody td {
    text-align: center;
}
tfoot td {
    padding-top: 1em;
    vertical-align: top;
}
td.pts a {
    font-size: small;
    padding: 0 1ex;
}
p#footer {
    font-size: small;
    color: gray;
    margin: 0;
}
div#control {
    text-align: right;
    position: fixed;
    right: 2em;
    font-size: small;
    width: 15em;
}
ul#legend {
    list-style: none;
}
ul#legend li {
    margin: 0.5ex 0;
    text-align: center;
}
ul#legend li span {
    display: block;
}
tr.arch-details {
    display: none;
}
tr.arch-details td {
    font-size: small;
    text-align: right;
}
tbody tr td.uptodate0 {
	background: #fcaf3e;
}

tbody tr td.uptodate1 {
	background: #fccc46;
}

tbody tr td.uptodate2 {
	background: #fce94f;
}

tbody tr td.uptodate3 {
	background: #8ae234;
}

p#summary {
    position: absolute;
    right: 1em;
    top: -1ex;
    color: white;
}
</style>
</head>
<body>
 <div id="header">
  <img id="logo" src="gnome-64.png" alt="GNOME">
  <h1>Status of GNOME %(version)s in Debian</h1>
 </div>
 <div id="content">

<div id="control">
<ul id="legend">
 <li><span class="ok-stable">Up-to-date in stable</span></li>
 <li><span class="ok-testing">Up-to-date in testing</span></li>
 <li><span class="ok">Up-to-date in unstable</span></li>
 <li><span class="ok-experimental">Up-to-date in experimental</span></li>
 <li><span class="waiting-in-new">Waiting in NEW</span></li>
 <li><span class="lagging-minor">Not up-to-date, lagging by a minor version</span></li>
 <li><span class="lagging-major">Not up-to-date, lagging by a major version</span></li>
 <li><span class="missing">Not in Debian</span></li>
</ul>

<p>
<input type="checkbox" id="archs" onclick="display_details()"><label for="archs">Display arch details</label>
</p>
</div>


<table>
<thead>
<tr><td></td> <td></td> <th>Upstream</th> <th>Debian</th> <th>Archs</th> </tr>
</thead>
<tbody>

'''

DEBIAN_NAME_MAPPING = {
    'glib': 'glib2.0',
    'GConf': 'gconf',
    'ORBit2': 'orbit2',
    'atk': 'atk1.0',
    'gtk+': 'gtk+2.0',
    'libIDL': 'libidl',
    'libart_lgpl': 'libart-lgpl',
    'libhandy': 'libhandy-1',
    'pango': 'pango1.0',
    'libglade': 'libglade2',
    'eel': 'eel2',
    'gtk-engines': 'gtk2-engines',
    'libgtop': 'libgtop2',
    'gstreamer': 'gstreamer0.10',
    'gst-plugins-base': 'gst-plugins-base0.10',
    'gst-plugins-good': 'gst-plugins-good0.10',
    'epiphany': 'epiphany-browser',
    'snapshot': 'gnome-snapshot',
    'gtkhtml': 'gtkhtml3.14',
    'orca': 'gnome-orca',
    'glade3': 'glade-3',
    'gnome-control-center': 'control-center',
    'libadwaita': 'libadwaita-1',
    # C++ bindings
    'gconfmm': 'gconfmm2.6',
    'glibmm': 'glibmm2.4',
    'gnome-vfsmm': 'gnome-vfsmm2.6',
    'gtkmm': 'gtkmm2.4',
    'libglademm': 'libglademm2.4',
    'libgnomecanvasmm': 'libgnomecanvasmm2.6',
    'libgnomemm': 'libgnomemm2.6',
    'libgnomeuimm': 'libgnomeuimm2.6',
    'libsigc++': 'libsigc++-2.0',
    'libxml++': 'libxml++2.6',
    'atkmm': 'atkmm1.6',
    # perl bindings
    'Glib': 'libglib-perl',
    'Gnome2': 'libgnome2-perl',
    'Gnome2-Canvas': 'libgnome2-canvas-perl',
    'Gnome2-GConf': 'libgnome2-gconf-perl',
    'Gnome2-VFS': 'libgnome2-vfs-perl',
    'Gtk2': 'libgtk2-perl',
    'Gtk2-GladeXML': 'libgtk2-gladexml-perl',
    'Pango': 'libpango-perl',
    # C# bindings
    'gtk-sharp': 'gtk-sharp2',
    'gnome-sharp': 'gnome-sharp2',
    'gnome-desktop-sharp': 'gnome-desktop-sharp2',
    # Telepathy stuff
    'stream-engine': 'telepathy-stream-engine',
    # Other stuff
    'calls': 'gnome-calls',
    'clutter': 'clutter-1.0',
    'dconf': 'd-conf',
    'NetworkManager': 'network-manager',
    'polkit-gnome': 'policykit-1-gnome',
    'cantarell-fonts': 'fonts-cantarell',
    'sushi': 'gnome-sushi',
    'ModemManager': 'modemmanager',
    'gom': 'libgom',
}

def command_cache_execute(cmd):
    if cmd[0] == 'rmadison':
        cmd[0] = '/home/fpeters/my-rmadison-2'
        #cmd.insert(1, 'udd')
        #cmd.insert(1, '--url')
        #print cmd
    s = hashlib.md5(repr(cmd).encode()).hexdigest()
    cache_file = os.path.join(CACHE_DIR, s)
    if os.path.exists(cache_file):
        return open(cache_file).read()
    command = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    st = command.communicate()[0].decode()
    if command.returncode == 0:
        open(cache_file, 'w').write(st)
    return st


def url_cache_read(url, prefix='', mode='t'):
    s = prefix + hashlib.md5(url.encode()).hexdigest()
    cache_file = os.path.join(CACHE_DIR, s)
    if os.path.exists(cache_file):
        with open(cache_file, 'r%s' % mode) as fd:
            return fd.read()
    try:
        resp = urllib.request.urlopen(url)
    except urllib.error.HTTPError:
        return None
    st = resp.read()
    if not os.path.exists(CACHE_DIR):
        os.mkdir(CACHE_DIR)
    if mode == 't':
        open(cache_file, 'w').write(st.decode())
        return st.decode()
    else:
        open(cache_file, 'wb').write(st)
        return st


def soap_cache_call(p, method, **kwargs):
    s = hashlib.md5(repr((method, kwargs))).hexdigest()
    cache_file = os.path.join(CACHE_DIR, s)
    if os.path.exists(cache_file) and (kwargs and kwargs.get('source') != 'accerciser'):
        try:
            return pickle.load(open(cache_file, 'rb'))
        except pickle.UnpicklingError:
            pass
        except EOFError:
            pass
    st = getattr(p, method)(**kwargs)
    pickle.dump(st, open(cache_file, 'wb'))
    return st

class Package:
    _all_debian_package_versions = None

    name = None
    upstream_version = None

    suite = None
    language = None # for the 'bindings' suite

    def __repr__(self):
        return '<Package name=%s, upstream:%s>' % (self.name, self.upstream_version)

    def get_debian_name(self):
        if self.name in ('gtk', 'gtk+') and self.upstream_version and (
                self.upstream_version.startswith('3.9') or
                self.upstream_version.startswith('4.')):
            return 'gtk4'
        if self.name in ('gtk', 'gtk+') and self.upstream_version and (
                self.upstream_version.startswith('2.9') or
                self.upstream_version.startswith('3.')):
            return 'gtk+3.0'
        if self.name == 'libwnck' and self.upstream_version and (
                self.upstream_version.startswith('2.9') or
                self.upstream_version.startswith('3.')):
            return 'libwnck3'
        if self.name == 'clutter-gst' and self.upstream_version and (
                self.upstream_version.startswith('3.')):
            return 'clutter-gst-3.0'
        if self.name.startswith('gst') and self.upstream_version and (
                self.upstream_version.startswith('1.0')):
            return DEBIAN_NAME_MAPPING.get(self.name, self.name).replace('0.10', '1.0')
        if self.name == 'gtksourceview' and self.upstream_version and (
                self.upstream_version.startswith('5.')):
            return 'gtksourceview5'
        if self.name == 'gtksourceview' and self.upstream_version and (
                self.upstream_version.startswith('3.9') or
                self.upstream_version.startswith('4.')):
            return 'gtksourceview4'
        if self.name == 'libsigc++' and self.upstream_version and (
                self.upstream_version.startswith('2.9') or
                self.upstream_version.startswith('3.')):
            return 'libsigc++-3.0'
        if self.name == 'gtkmm' and self.upstream_version and (
                self.upstream_version.startswith('3.9') or
                self.upstream_version.startswith('4.')):
            return 'gtkmm4.0'
        if self.name == 'libsoup' and self.upstream_version and (
                self.upstream_version.startswith('3.')):
            return 'libsoup3'
        if self.name == 'gnome-bluetooth' and self.upstream_version and (
                int(self.upstream_version.split('.')[0]) > 40):
            return 'gnome-bluetooth3'
        if self.name == 'gcr' and self.upstream_version and (
                int(self.upstream_version.split('.')[0]) >= 4):
            return 'gcr4'
        if self.name == 'libpeas' and self.upstream_version and (
                self.upstream_version.startswith('1.9') or
                int(self.upstream_version.split('.')[0]) >= 2):
            return 'libpeas2'
        if self.name == 'glibmm' and self.upstream_version and (
                (int(self.upstream_version.split('.')[0]), int(self.upstream_version.split('.')[1])) >= (2, 68)):
            return 'glibmm2.68'
        if self.name == 'pangomm' and self.upstream_version and (
                (int(self.upstream_version.split('.')[0]), int(self.upstream_version.split('.')[1])) >= (2, 48)):
            return 'pangomm2.48'
        return DEBIAN_NAME_MAPPING.get(self.name, self.name)
    debian_name = property(get_debian_name)

    _available_packages = None
    def get_available_packages(self):
        if self._available_packages:
            return self._available_packages

        # look for source packages
        debian_packages = None
        import copy
        for package_key, package in self._all_debian_package_versions.items():
            if package_key in ('gtk-update-icon-cache',):
                # ignore packages created by multiple versions
                continue
            if not set(package.keys()).intersection(['unstable', 'experimental', 'new']):
                continue
            for suite in package.values():
                for version in suite.values():
                    if version.get('source') == self.debian_name:
                        if debian_packages:
                            for suite2 in package.keys():
                                if not suite2 in debian_packages:
                                    debian_packages[suite2] = package[suite2]
                                else:
                                    for version2 in package[suite2].keys():
                                        debian_packages[suite2][version2] = package[suite2][version2]
                        else:
                            debian_packages = copy.deepcopy(package)
                else:
                    continue
                break
            else:
                continue
            break
        if not debian_packages:
            debian_packages = self._all_debian_package_versions.get(self.debian_name)
        if not debian_packages:
            return []
            sys.exit(1)
            return self.fallback_get_available_packages()

        packages = []
        for suite in ('stable', 'testing', 'unstable', 'experimental', 'new'):
            for version in debian_packages.get(suite, {}):
                archs = [x for x in debian_packages[suite][version]['architectures']]
                if archs != ['source']:
                    archs = [x for x in archs if x != 'source']
                packages.append((version, suite, archs))
        def key_func():
            return functools.cmp_to_key(apt_pkg.version_compare)
        packages.sort(key=lambda x: key_func()(x[0]))
        #print self.debian_name
        #print 'packages:', packages
        return packages
            

    def fallback_get_available_packages(self):
        packages = []
        okayish_versions = {}
        for line in reversed(command_cache_execute(
                    ['rmadison', '-S', self.debian_name]).splitlines()):
            p, v, s, a = [x.strip() for x in line.split('|')]
            if '+b' in v:
                v = v[:v.index('+b')]
            if 'source' in a:
                okayish_versions[v] = True

        for line in reversed(command_cache_execute(
                    ['rmadison', '-S', self.debian_name]).splitlines()):
            p, v, s, a = [x.strip() for x in line.split('|')]
            if '+b' in v:
                v = v[:v.index('+b')]
            if not v in okayish_versions:
                continue
            if a == 'source':
                continue

            archs = [x.strip() for x in a.split(',') if x.strip() != 'source']
            if (v, s) in [(x[0], x[1]) for x in packages]:
                # already got a version number for that suite
                v1, s1, a1 = [x for x in packages if x[1] == s][0]

                # when a binary all package was selected, but there is another
                # binary package that is arch specific, prefer it.
                if not 'all' in a1:
                    continue
                if 'all' in a1 and 'all' in archs:
                    continue
                packages.remove((v1, s1, a1))
            packages.append((v, s, archs))
        packages.sort(lambda x,y: apt_pkg.version_compare(x[0], y[0]))
        self._available_packages = packages
        print(packages)
        sys.exit(1)
        return packages

    def get_debian_version(self, debian_suite):
        versions = {}
        for version, suite, archs in self.get_available_packages():
            versions[suite] = version
        return versions.get(debian_suite)

    def get_debian_archs(self, package_version):
        for version, suite, archs in self.get_available_packages():
            if version == package_version:
                return archs
        return []

    def get_debian_team(self):
        pts_soap_url = 'http://packages.qa.debian.org/cgi-bin/soap-alpha.cgi'
        if ZSI:
            p = ZSI.client.NamedParamBinding(url=pts_soap_url)
        elif SOAPpy:
            p = SOAPpy.SOAPProxy(pts_soap_url)
        else:
            return '?'
        emails = [soap_cache_call(p, 'maintainer_email', source=self.debian_name)]
        uploader_emails = soap_cache_call(p, 'uploader_emails', source=self.debian_name)
        if uploader_emails and len(uploader_emails):
            if type(uploader_emails) is dict:
                emails.extend(uploader_emails.values())
            elif type(uploader_emails) is str:
                emails.append(uploader_emails)
            else:
                emails.extend(uploader_emails)
        known_teams = (
            ('pkg-gnome-maintainers', 'GNOME'),
            ('pkg-telepathy-maintainers', 'Telepathy'),
            ('pkg-evolution-maintainers', 'Evolution'),
            ('pkg-perl-maintainers', 'Perl'),
            ('pkg-cli-libs-team', 'CLI'),
            ('pkg-mono-group', 'Mono'),
            ('pkg-gstreamer-maintainers', 'GStreamer'),
        )
        for team_email, team_name in known_teams:
            try:
                if [x for x in emails if x.startswith(team_email)]:
                     return team_name
            except AttributeError:
                pass
        return '-'
    debian_team = property(get_debian_team)
    
    def get_debian_pts_url(self):
        #return 'http://packages.qa.debian.org/common/index.html?src=%s' % urllib.parse.quote(self.debian_name)
        return 'https://tracker.debian.org/pkg/%s' % urllib.parse.quote(self.debian_name)
    debian_pts_url = property(get_debian_pts_url)

    def get_status_and_debian_version(self):
        suite_version = None
        for debsuite in (STABLE, TESTING, UNSTABLE, 'experimental', 'incoming', 'new', 'vcs'):
            suite_version = self.get_debian_version(debsuite) or suite_version
            if vcmp(self.get_debian_version(debsuite), self.upstream_version) >= 0:
                if debsuite == STABLE:
                    status = 'ok-stable'
                elif debsuite == TESTING:
                    status = 'ok-testing'
                elif debsuite == UNSTABLE:
                    status = 'ok'
                elif debsuite == 'new':
                    status = 'waiting-in-new'
                else:
                    status = 'notinsid'
                return (status, suite_version)
        else:
            if suite_version and suite_version == self.get_debian_version('new'):
                status = 'missing'
                return ('waiting-in-new', suite_version)
            elif suite_version:
                # package is available, but outdated
                if get_major_version(suite_version) == get_major_version(self.upstream_version):
                    status = 'minor'
                else:
                    status = 'major'
                return (status, suite_version)
            else:
                # package is not available
                status = 'missing'
                return (status, None)

    def print_tr(self, output):
        tr_status, suite_version = self.get_status_and_debian_version()
        if suite_version:
            td_debian_version = '<td>%s</td>' % suite_version
        else:
            td_debian_version = '<td>missing</td>'

        print('<tr class="%s">' % tr_status, file=output)
        if self.language:
            print('<th>%s / %s</th>' % (self.language, self.debian_name), file=output)
        else:
            print('<th>%s</th>' % self.debian_name, file=output)
        if tr_status != 'missing' and tr_status != 'waiting-in-new':
            print('<td class="pts"><a href="%s">PTS</a></td>' % self.debian_pts_url, file=output)
        else:
            print('<td class="pts"></td>', file=output)
        print('<td>%s</td>' % self.upstream_version, file=output)
        if suite_version:
            print(td_debian_version, file=output)
            archs = self.get_debian_archs(suite_version)
            if 'all' in archs:
                print('<td>all</td>', file=output)
            else:
                packages = [x for x in self.get_available_packages() if x[1] in ('unstable', 'experimental', 'new')]
                if len(packages) == 1:
                    print('<td>all</td>', file=output)
                else:
                    all_archs = {}
                    for version, debsuite, archlist in packages:
                        for a in archlist:
                            all_archs[a] = True
                    aclass_name = ''
                    if tr_status in ('ok', 'ok-testing') and len(all_archs):
                        aclass_name = 'uptodate%d' % int(3.0*len(archs)/len(all_archs))
                    if tr_status == 'notinsid' and len(all_archs):
                        aclass_name = 'uptodate%d' % int(2.0*len(archs)/len(all_archs))
                    print('<td title="%s" class="%s">%s/%s</td>' % (
                            ', '.join(archs), aclass_name, len(archs), len(all_archs)), file=output)
        print('</tr>', file=output)

    def print_arch_details_tr(self, output):
        packages = self.get_available_packages()
        if not packages:
            return
        if 'all' in packages[-1][-1]:
            return
        packages = [x for x in packages if x[1] in ('unstable', 'experimental', 'new')]
        if len(packages) <= 1:
            return
        print('<tr class="arch-details"><td colspan="5">', file=output)
        for version, debsuite, archs in packages:
            print(debsuite, 'has', version, 'for', ', '.join(archs), file=output)
            print('<br>', file=output)
        print('</td></tr>', file=output)



def get_modules_at_url(url, suite):
    r = []
    s = url_cache_read(url)
    for module in re.findall(r'<a href="(.*tar\.xz)">', s):
        p = Package()
        p.suite = suite
        name, version = re.findall('(.*?)-([(\d+|alpha|beta|rc)\.]+[\w]*).tar.xz', module)[0]
        name = urllib.parse.unquote(name)
        if (name, version) == ('vte', '0.43.0'):
            # XXX: hack around wrong vte version being pulled upstream in 3.18.2
            version = '0.42.1'
        p.name, p.upstream_version = name, version
        r.append(p)
    for module in re.findall(r'<a href="(.*tar\.bz2)">', s):
        p = Package()
        p.suite = suite
        name, version = re.findall('(.*)-([\d+\.]+[\w]*).tar.bz2', module)[0]
        if [x for x in r if x.name == name]:
            continue
        p.name, p.upstream_version = name, version
        r.append(p)
    return r

def get_gnome_suites(gnome_version):
    suites = ['platform', 'desktop', 'admin', 'bindings']
    if float(gnome_version) > 2.16:
        suites.append('devtools')
    if float(gnome_version) > 2.90:
        suites = ['core', 'apps']
    if float(gnome_version) > 3.29:
        suites = ['core']
    if float(gnome_version) >= 46:
        suites = ['sdk', 'core-deps', 'core']
    return suites


def get_gnome_packages(gnome_version):
    # get latest minor version
    s = url_cache_read('https://ftp.gnome.org/pub/GNOME/teams/releng/')
    minors = re.findall(r'<a href="%s(|\.\d+|\.alpha|\.beta|\.rc)/".*?>%s' % (gnome_version, gnome_version), s)
    minors.sort(key=lambda x: int(x.replace('.', '').replace('rc', '-1').replace('beta', '-2').replace('alpha', '-3') or 0))
    # -> ['.alpha', '.beta', '.rc', '', '.2', '.3']
    try:
        last_minor = minors[-1]
    except IndexError:  # unreleased version
        if gnome_version == '3.0':
            devel_gnome_version = '2.91'
        else:
            t = gnome_version.split('.')
            t[-1] = str(int(t[-1])-1)
            devel_gnome_version = '.'.join(t)
        return get_gnome_packages(devel_gnome_version)

    # get BuildStream snapshot
    #print(f'https://download.gnome.org/teams/releng/{gnome_version}{last_minor}/gnome-{gnome_version}{last_minor}.tar.xz')
    s1 = url_cache_read(f'https://download.gnome.org/teams/releng/{gnome_version}{last_minor}/gnome-{gnome_version}{last_minor}.tar.xz', mode='b')
    if s1 is None:
        #print(f'https://download.gnome.org/sources/gnome-build-meta/47/gnome-build-meta-{gnome_version}{last_minor}.tar.xz')
        s1 = url_cache_read(f'https://download.gnome.org/sources/gnome-build-meta/{gnome_version}/gnome-build-meta-{gnome_version}{last_minor}.tar.xz', mode='b')
    if s1 is None:
        return []

    gnome_packages = []
    tf = tarfile.open(fileobj=io.BytesIO(s1))
    for suite in get_gnome_suites(gnome_version):
        for member in [x for x in tf.getmembers() if f'elements/{suite}/' in x.path]:
            details = tf.extractfile(member).read().decode()
            if 'url: gnome_downloads' not in details:
                continue
            try:
                name, version = re.findall(r'url: .*/(.*)-([\d+\.]+.*).tar.[gx]z', details)[0]
            except IndexError:
                print('not found:', member)
                continue
            p = Package()
            p.suite = suite
            p.name, p.upstream_version = name, version
            gnome_packages.append(p)
    
    return gnome_packages


TELEPATHY_WHITELIST = ['telepathy-farstream',
        'telepathy-glib', 'telepathy-haze', 'telepathy-idle',
        'telepathy-mission-control',
        'telepathy-qt', 'telepathy-rakia',
        'telepathy-spec'
]
	

def get_telepathy_packages():
    telepathy_releases_url = 'http://telepathy.freedesktop.org/releases/'
    s = url_cache_read(telepathy_releases_url)
    telepathy_packages = []
    for module in re.findall(r'folder.gif.*<a href="(.*)/">', s):
        if module not in TELEPATHY_WHITELIST:
            continue
        if module == 'gst-plugins-farsight':
            url = 'http://farsight.freedesktop.org/releases/gst-plugins-farsight/'
        else:
            url = telepathy_releases_url + module + '/'
        s = url_cache_read(url)
        p = Package()
        p.suite = 'telepathy'
        try:
            versions = re.findall('.*-([\d+\.]+).tar.gz', s)
            versions = [x for x in versions if not '.99' in x]
            version = versions[-1]
        except IndexError:
            continue
        p.name, p.upstream_version = module, version
        telepathy_packages.append(p)
    return telepathy_packages

def vcmp(v1, v2):
    '''compare two versions, without debian artefacts'''
    if v1 is None and v2 is None:
        return 0
    if v1 is None:
        return -1
    if v2 is None:
        return 1

    def clean_v(x):
        if ':' in x:
            x = x[x.index(':')+1:]
        x = x.replace('.alpha', '~alpha').replace('.beta', '~beta').replace('.rc', '~rc')
        return x

    return apt_pkg.version_compare(clean_v(v1), clean_v(v2))

def get_major_version(v):
    if ':' in v:
        v = v[v.index(':')+1:]
    return '.'.join(v.split('.')[:2]).replace('.alpha', '').replace('.beta', '').replace('.rc', '')

def amend_debian_name_mapping(version):
    # some packages changed name over gnome versions
    if apt_pkg.version_compare(version, '2.19') > 0:
        DEBIAN_NAME_MAPPING.update({'gtksourceview': 'gtksourceview2'})
    if apt_pkg.version_compare(version, '2.21') > 0:
        DEBIAN_NAME_MAPPING.update({'libsoup': 'libsoup2.4'})
    if apt_pkg.version_compare(version, '2.26') > 0:
        DEBIAN_NAME_MAPPING.update({'telepathy-mission-control': 'telepathy-mission-control-5'})
    if apt_pkg.version_compare(version, '2.29') > 0:
        DEBIAN_NAME_MAPPING.update({'gdm': 'gdm3'})
    if apt_pkg.version_compare(version, '2.91') > 0:
        # was control-center previously
        DEBIAN_NAME_MAPPING.update({'gnome-control-center': 'gnome-control-center'})
        DEBIAN_NAME_MAPPING.update({'gnome-desktop': 'gnome-desktop3'})
        #DEBIAN_NAME_MAPPING.update({'evolution-data-server': 'evolution-data-server3'})
        DEBIAN_NAME_MAPPING.update({'gtkmm': 'gtkmm3.0'})
        #DEBIAN_NAME_MAPPING.update({'libgweather': 'libgweather3'})
        DEBIAN_NAME_MAPPING.update({'vte': 'vte3'})
        DEBIAN_NAME_MAPPING.update({'libunique': 'libunique3'})
    if apt_pkg.version_compare(version, '3.1') > 0:
        DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.16'})
        DEBIAN_NAME_MAPPING.update({'gtksourceview': 'gtksourceview3'})
        DEBIAN_NAME_MAPPING.update({'rest': 'librest'})
    if apt_pkg.version_compare(version, '3.7') > 0:
        DEBIAN_NAME_MAPPING.update({'clutter-gst': 'clutter-gst-2.0'})
        DEBIAN_NAME_MAPPING.update({'libgee': 'libgee-0.8'})
        DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.20'})
    if apt_pkg.version_compare(version, '3.9') > 0:
        DEBIAN_NAME_MAPPING.update({'gstreamer': 'gstreamer1.0'})
        DEBIAN_NAME_MAPPING.update({'gst-plugins-base': 'gst-plugins-base1.0'})
        DEBIAN_NAME_MAPPING.update({'gst-plugins-good': 'gst-plugins-good1.0'})
        DEBIAN_NAME_MAPPING.update({'gst-plugins-bad': 'gst-plugins-bad1.0'})
        DEBIAN_NAME_MAPPING.update({'gst-plugins-ugly': 'gst-plugins-ugly1.0'})
        DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.22'})
    if apt_pkg.version_compare(version, '3.11') > 0:
        DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.24'})
    if apt_pkg.version_compare(version, '3.13') > 0:
        DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.26'})
        DEBIAN_NAME_MAPPING.update({'vte': 'vte2.91'})
    if apt_pkg.version_compare(version, '3.15') > 0:
        DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.28'})
    if apt_pkg.version_compare(version, '3.17') > 0:
        DEBIAN_NAME_MAPPING.update({'vala': 'vala'})
    if apt_pkg.version_compare(version, '3.25') > 0:
        DEBIAN_NAME_MAPPING.update({'dconf': 'dconf'})
        DEBIAN_NAME_MAPPING.update({'orca': 'orca'})
    if apt_pkg.version_compare(version, '42~') > 0:
        DEBIAN_NAME_MAPPING.update({'libgweather': 'libgweather4'})
        DEBIAN_NAME_MAPPING.update({'gnome-desktop': 'gnome-desktop'})


if __name__ == '__main__':
    if not os.path.exists(CACHE_DIR):
        os.mkdir(CACHE_DIR)

    parser = OptionParser()
    parser.add_option('--version', dest='version', metavar='VERSION', default='2.26')
    parser.add_option('--output', dest='output', metavar='OUTPUT', default='-')
    options, args = parser.parse_args()

    version = options.version
    amend_debian_name_mapping(version)


    if options.output == '-':
        output = sys.stdout
    else:
        output = io.StringIO()

    print(HTML_AT_TOP % {'version': version}, file=output)

    gnome_packages = get_gnome_packages(version)
    telepathy_packages = []  # get_telepathy_packages()
    modules = gnome_packages + telepathy_packages
    if not modules:
       print('failed to get any modules for %s' % version)
       sys.exit(1)

    Package._all_debian_package_versions = {}
    for i in range(0, len(modules), 10):
        sliced_modules = modules[i:i+10]
        url = 'https://api.ftp-master.debian.org/madison?f=json&S=true&package=%s' % urllib.parse.quote(' '.join(
                [x.debian_name for x in sliced_modules]))
        #print(url)
        #Package._all_debian_package_versions.update(json.load(urllib.parse.urlopen(url, context=context))[0])
        Package._all_debian_package_versions.update(
                json.loads(command_cache_execute(['curl', '-k', '--silent', url]))[0])

    if not gnome_packages:
        print('<tr><td colspan="5">Getting this script working on GNOME release days is hard; let\'s go shopping!</td></tr>', file=output)

    for suite in get_gnome_suites(version):
        print('<tr><td class="heading" colspan="5">%s</td></tr>' % suite, file=output)
        suite_modules = [x for x in modules if x.suite == suite]
        suite_modules.sort(key=lambda x: (x.language, x.name.lower()))
        for module in suite_modules:
            module.print_tr(output)
            module.print_arch_details_tr(output)

    if telepathy_packages:
        print('<tr><td class="heading" colspan="5">%s</td></tr>' % 'Telepathy', file=output)
        for module in telepathy_packages:
            module.print_tr(output)
            module.print_arch_details_tr(output)

    print('</tbody>', file=output)
    print('</table>', file=output)

    total_modules = len(modules)
    ok_testing = len([x for x in modules if x.get_status_and_debian_version()[0] in ('ok-testing', 'ok-stable')])
    ok_unstable = len([x for x in modules if x.get_status_and_debian_version()[0] in ('ok', 'ok-testing', 'ok-stable')])

    print('<p id="summary">', file=output)
    print('%s out of %s packages are up-to-date in unstable (%d%%)' % (
            ok_unstable, total_modules, 100.*ok_unstable/total_modules), file=output)
    print('<br/>%s out of %s packages are up-to-date in testing (%d%%)' % (
            ok_testing, total_modules, 100.*ok_testing/total_modules), file=output)
    print('</p>', file=output)

    print('</div>', file=output)

    print('<p id="footer">', file=output)
    print('Generated:', time.strftime('%Y-%m-%d %H:%M:%S %z'), file=output)
    print('on ', socket.getfqdn(), file=output)
    print('by fpeters (contact: fpeters at debian dot org)', file=output)
    print(', <a href="https://people.debian.org/~fpeters/dgs3.py">generation script</a>', file=output)
    print('</p>', file=output)

    print('</html>', file=output)

    if options.output != '-':
        open(options.output, 'w').write(output.getvalue())
