#! /usr/bin/env python # # dgs.py -- generation script for the GNOME in Debian status page # Copyright (C) 2008-2011 Frederic Peters # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public # License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import hashlib import urllib import urllib2 import re import os import sys import apt_pkg import subprocess import time import socket import cPickle import json from optparse import OptionParser from cStringIO import StringIO import ssl context = ssl._create_unverified_context() STABLE = 'jessie' TESTING = 'stretch' UNSTABLE = 'sid' STABLE = 'stable' TESTING = 'testing' UNSTABLE = 'unstable' try: import ZSI.client except ImportError: ZSI = None try: import SOAPpy except ImportError: SOAPpy = None apt_pkg.init() CACHE_DIR = 'debian-gnome-status-cache' HTML_AT_TOP = ''' Status of GNOME %(version)s in Debian
  • Up-to-date in testing
  • Up-to-date in unstable
  • Up-to-date in experimental
  • Waiting in NEW
  • Not up-to-date, lagging by a minor version
  • Not up-to-date, lagging by a major version
  • Not in Debian

''' DEBIAN_NAME_MAPPING = { 'glib': 'glib2.0', 'GConf': 'gconf', 'ORBit2': 'orbit2', 'atk': 'atk1.0', 'gtk+': 'gtk+2.0', 'libIDL': 'libidl', 'libart_lgpl': 'libart-lgpl', 'pango': 'pango1.0', 'libglade': 'libglade2', 'eel': 'eel2', 'gtk-engines': 'gtk2-engines', 'libgtop': 'libgtop2', 'gstreamer': 'gstreamer0.10', 'gst-plugins-base': 'gst-plugins-base0.10', 'gst-plugins-good': 'gst-plugins-good0.10', 'epiphany': 'epiphany-browser', 'gtkhtml': 'gtkhtml3.14', 'orca': 'gnome-orca', 'glade3': 'glade-3', 'gnome-control-center': 'control-center', # C++ bindings 'gconfmm': 'gconfmm2.6', 'glibmm': 'glibmm2.4', 'gnome-vfsmm': 'gnome-vfsmm2.6', 'gtkmm': 'gtkmm2.4', 'libglademm': 'libglademm2.4', 'libgnomecanvasmm': 'libgnomecanvasmm2.6', 'libgnomemm': 'libgnomemm2.6', 'libgnomeuimm': 'libgnomeuimm2.6', 'libsigc++': 'libsigc++-2.0', 'libxml++': 'libxml++2.6', 'atkmm': 'atkmm1.6', # perl bindings 'Glib': 'libglib-perl', 'Gnome2': 'libgnome2-perl', 'Gnome2-Canvas': 'libgnome2-canvas-perl', 'Gnome2-GConf': 'libgnome2-gconf-perl', 'Gnome2-VFS': 'libgnome2-vfs-perl', 'Gtk2': 'libgtk2-perl', 'Gtk2-GladeXML': 'libgtk2-gladexml-perl', 'Pango': 'libpango-perl', # C# bindings 'gtk-sharp': 'gtk-sharp2', 'gnome-sharp': 'gnome-sharp2', 'gnome-desktop-sharp': 'gnome-desktop-sharp2', # Telepathy stuff 'stream-engine': 'telepathy-stream-engine', 'clutter': 'clutter-1.0', 'dconf': 'd-conf', 'NetworkManager': 'network-manager', 'polkit-gnome': 'policykit-1-gnome', 'cantarell-fonts': 'fonts-cantarell', 'sushi': 'gnome-sushi', 'ModemManager': 'modemmanager', 'gom': 'libgom', } def command_cache_execute(cmd): if cmd[0] == 'rmadison': cmd[0] = '/home/fpeters/my-rmadison-2' #cmd.insert(1, 'udd') #cmd.insert(1, '--url') #print cmd s = hashlib.md5(repr(cmd)).hexdigest() cache_file = os.path.join(CACHE_DIR, s) if os.path.exists(cache_file): return file(cache_file).read() command = subprocess.Popen(cmd, stdout=subprocess.PIPE) st = command.communicate()[0] if command.returncode == 0: file(cache_file, 'w').write(st) return st def url_cache_read(url, prefix = ''): s = prefix + hashlib.md5(url).hexdigest() cache_file = os.path.join(CACHE_DIR, s) if os.path.exists(cache_file): return file(cache_file).read() try: st = urllib2.urlopen(url).read() except: return '' if not os.path.exists(CACHE_DIR): os.mkdir(CACHE_DIR) file(cache_file, 'w').write(st) return st def soap_cache_call(p, method, **kwargs): s = hashlib.md5(repr((method, kwargs))).hexdigest() cache_file = os.path.join(CACHE_DIR, s) if os.path.exists(cache_file) and (kwargs and kwargs.get('source') != 'accerciser'): try: return cPickle.load(file(cache_file)) except cPickle.UnpicklingError: pass except EOFError: pass st = getattr(p, method)(**kwargs) cPickle.dump(st, file(cache_file, 'w')) return st class Package: _all_debian_package_versions = None name = None upstream_version = None suite = None language = None # for the 'bindings' suite def __repr__(self): return '' % (self.name, self.upstream_version) def get_debian_name(self): if self.name == 'gtk+' and self.upstream_version and ( self.upstream_version.startswith('2.9') or self.upstream_version.startswith('3.')): return 'gtk+3.0' if self.name == 'libwnck' and self.upstream_version and ( self.upstream_version.startswith('2.9') or self.upstream_version.startswith('3.')): return 'libwnck3' if self.name == 'clutter-gst' and self.upstream_version and ( self.upstream_version.startswith('3.')): return 'clutter-gst-3.0' if self.name.startswith('gst') and self.upstream_version and ( self.upstream_version.startswith('1.0')): return DEBIAN_NAME_MAPPING.get(self.name, self.name).replace('0.10', '1.0') return DEBIAN_NAME_MAPPING.get(self.name, self.name) debian_name = property(get_debian_name) _available_packages = None def get_available_packages(self): if self._available_packages: return self._available_packages # look for source packages debian_packages = None import copy for package in self._all_debian_package_versions.values(): if not set(package.keys()).intersection(['unstable', 'experimental', 'new']): continue for suite in package.values(): for version in suite.values(): if version.get('source') == self.debian_name: if debian_packages: for suite2 in package.keys(): if not suite2 in debian_packages: debian_packages[suite2] = package[suite2] else: for version2 in package[suite2].keys(): debian_packages[suite2][version2] = package[suite2][version2] else: debian_packages = copy.deepcopy(package) else: continue break else: continue break if not debian_packages: debian_packages = self._all_debian_package_versions.get(self.debian_name) if not debian_packages: return [] sys.exit(1) return self.fallback_get_available_packages() packages = [] for suite in ('testing', 'unstable', 'experimental', 'new'): for version in debian_packages.get(suite, {}): archs = [x for x in debian_packages[suite][version]['architectures']] if archs != ['source']: archs = [x for x in archs if x != 'source'] packages.append((version, suite, archs)) packages.sort(lambda x,y: apt_pkg.version_compare(x[0], y[0])) #print self.debian_name #print 'packages:', packages return packages def fallback_get_available_packages(self): packages = [] okayish_versions = {} for line in reversed(command_cache_execute( ['rmadison', '-S', self.debian_name]).splitlines()): p, v, s, a = [x.strip() for x in line.split('|')] if '+b' in v: v = v[:v.index('+b')] if 'source' in a: okayish_versions[v] = True for line in reversed(command_cache_execute( ['rmadison', '-S', self.debian_name]).splitlines()): p, v, s, a = [x.strip() for x in line.split('|')] if '+b' in v: v = v[:v.index('+b')] if not v in okayish_versions: continue if a == 'source': continue archs = [x.strip() for x in a.split(',') if x.strip() != 'source'] if (v, s) in [(x[0], x[1]) for x in packages]: # already got a version number for that suite v1, s1, a1 = [x for x in packages if x[1] == s][0] # when a binary all package was selected, but there is another # binary package that is arch specific, prefer it. if not 'all' in a1: continue if 'all' in a1 and 'all' in archs: continue packages.remove((v1, s1, a1)) packages.append((v, s, archs)) packages.sort(lambda x,y: apt_pkg.version_compare(x[0], y[0])) self._available_packages = packages print packages sys.exit(1) return packages def get_debian_version(self, debian_suite): versions = {} for version, suite, archs in self.get_available_packages(): versions[suite] = version return versions.get(debian_suite) def get_debian_archs(self, package_version): for version, suite, archs in self.get_available_packages(): if version == package_version: return archs return [] def get_debian_team(self): pts_soap_url = 'http://packages.qa.debian.org/cgi-bin/soap-alpha.cgi' if ZSI: p = ZSI.client.NamedParamBinding(url=pts_soap_url) elif SOAPpy: p = SOAPpy.SOAPProxy(pts_soap_url) else: return '?' emails = [soap_cache_call(p, 'maintainer_email', source=self.debian_name)] uploader_emails = soap_cache_call(p, 'uploader_emails', source=self.debian_name) if uploader_emails and len(uploader_emails): if type(uploader_emails) is dict: emails.extend(uploader_emails.values()) elif type(uploader_emails) is str: emails.append(uploader_emails) else: emails.extend(uploader_emails) known_teams = ( ('pkg-gnome-maintainers', 'GNOME'), ('pkg-telepathy-maintainers', 'Telepathy'), ('pkg-evolution-maintainers', 'Evolution'), ('pkg-perl-maintainers', 'Perl'), ('pkg-cli-libs-team', 'CLI'), ('pkg-mono-group', 'Mono'), ('pkg-gstreamer-maintainers', 'GStreamer'), ) for team_email, team_name in known_teams: try: if [x for x in emails if x.startswith(team_email)]: return team_name except AttributeError: pass return '-' debian_team = property(get_debian_team) def get_debian_pts_url(self): #return 'http://packages.qa.debian.org/common/index.html?src=%s' % urllib2.quote(self.debian_name) return 'https://tracker.debian.org/pkg/%s' % urllib2.quote(self.debian_name) debian_pts_url = property(get_debian_pts_url) def get_status_and_debian_version(self): suite_version = None for debsuite in (TESTING, UNSTABLE, 'experimental', 'incoming', 'new', 'vcs'): suite_version = self.get_debian_version(debsuite) or suite_version if vcmp(self.get_debian_version(debsuite), self.upstream_version) >= 0: if debsuite == TESTING: status = 'ok-testing' elif debsuite == UNSTABLE: status = 'ok' elif debsuite == 'new': status = 'waiting-in-new' else: status = 'notinsid' return (status, suite_version) else: if suite_version and suite_version == self.get_debian_version('new'): status = 'missing' return ('waiting-in-new', suite_version) elif suite_version: # package is available, but outdated if get_major_version(suite_version) == get_major_version(self.upstream_version): status = 'minor' else: status = 'major' return (status, suite_version) else: # package is not available status = 'missing' return (status, None) def print_tr(self, output): tr_status, suite_version = self.get_status_and_debian_version() if suite_version: td_debian_version = '' % suite_version else: td_debian_version = '' print >> output, '' % tr_status if self.language: print >> output, '' % (self.language, self.name) else: print >> output, '' % self.name if tr_status != 'missing' and tr_status != 'waiting-in-new': print >> output, '' % self.debian_pts_url else: print >> output, '' print >> output, '' % self.upstream_version if suite_version: print >> output, td_debian_version archs = self.get_debian_archs(suite_version) if 'all' in archs: print >> output, '' else: packages = [x for x in self.get_available_packages() if x[1] in ('unstable', 'experimental', 'new')] if len(packages) == 1: print >> output, '' else: all_archs = {} for version, debsuite, archlist in packages: for a in archlist: all_archs[a] = True aclass_name = '' if tr_status in ('ok', 'ok-testing') and len(all_archs): aclass_name = 'uptodate%d' % int(3.0*len(archs)/len(all_archs)) if tr_status == 'notinsid' and len(all_archs): aclass_name = 'uptodate%d' % int(2.0*len(archs)/len(all_archs)) print >> output, '' % ( ', '.join(archs), aclass_name, len(archs), len(all_archs)) print >> output, '' def print_arch_details_tr(self, output): packages = self.get_available_packages() if not packages: return if 'all' in packages[-1][-1]: return packages = [x for x in packages if x[1] in ('unstable', 'experimental', 'new')] if len(packages) <= 1: return print >> output, '' def get_modules_at_url(url, suite): r = [] s = url_cache_read(url) for module in re.findall(r' 2.16: suites.append('devtools') if float(gnome_version) > 2.90: suites = ['core', 'apps'] return suites def get_gnome_packages(gnome_version): gnome_packages = [] # get latest minor version s = url_cache_read('http://ftp.gnome.org/pub/GNOME/teams/releng/') minors = re.findall(r'%s' % (gnome_version, gnome_version), s) try: last_minor = minors[-1] except IndexError: # unreleased version if gnome_version == '3.0': devel_gnome_version = '2.91' else: t = gnome_version.split('.') t[-1] = str(int(t[-1])-1) devel_gnome_version = '.'.join(t) return get_gnome_packages(devel_gnome_version) for suite in get_gnome_suites(gnome_version): url = 'http://ftp.gnome.org/pub/GNOME/%s/%s/%s.%s/sources/' % ( suite, gnome_version, gnome_version, last_minor) if suite == 'bindings': s = url_cache_read(url) for subdir in re.findall(r'', s): if module not in TELEPATHY_WHITELIST: continue if module == 'gst-plugins-farsight': url = 'http://farsight.freedesktop.org/releases/gst-plugins-farsight/' else: url = telepathy_releases_url + module + '/' s = url_cache_read(url) p = Package() p.suite = 'telepathy' try: versions = re.findall('.*-([\d+\.]+).tar.gz', s) versions = [x for x in versions if not '.99' in x] version = versions[-1] except IndexError: continue p.name, p.upstream_version = module, version telepathy_packages.append(p) return telepathy_packages def vcmp(v1, v2): '''compare two versions, without debian artefacts''' if v1 is None and v2 is None: return 0 if v1 is None: return -1 if v2 is None: return 1 if ':' in v1: v1 = v1[v1.index(':')+1:] if ':' in v2: v2 = v2[v2.index(':')+1:] return apt_pkg.version_compare(v1, v2) def get_major_version(v): if ':' in v: v = v[v.index(':')+1:] return '.'.join(v.split('.')[:2]) def amend_debian_name_mapping(version): # some packages changed name over gnome versions if apt_pkg.version_compare(version, '2.19') > 0: DEBIAN_NAME_MAPPING.update({'gtksourceview': 'gtksourceview2'}) if apt_pkg.version_compare(version, '2.21') > 0: DEBIAN_NAME_MAPPING.update({'libsoup': 'libsoup2.4'}) if apt_pkg.version_compare(version, '2.26') > 0: DEBIAN_NAME_MAPPING.update({'telepathy-mission-control': 'telepathy-mission-control-5'}) if apt_pkg.version_compare(version, '2.29') > 0: DEBIAN_NAME_MAPPING.update({'gdm': 'gdm3'}) if apt_pkg.version_compare(version, '2.91') > 0: # was control-center previously DEBIAN_NAME_MAPPING.update({'gnome-control-center': 'gnome-control-center'}) DEBIAN_NAME_MAPPING.update({'gnome-desktop': 'gnome-desktop3'}) #DEBIAN_NAME_MAPPING.update({'evolution-data-server': 'evolution-data-server3'}) DEBIAN_NAME_MAPPING.update({'gtkmm': 'gtkmm3.0'}) #DEBIAN_NAME_MAPPING.update({'libgweather': 'libgweather3'}) DEBIAN_NAME_MAPPING.update({'vte': 'vte3'}) DEBIAN_NAME_MAPPING.update({'libunique': 'libunique3'}) if apt_pkg.version_compare(version, '3.1') > 0: DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.16'}) DEBIAN_NAME_MAPPING.update({'gtksourceview': 'gtksourceview3'}) DEBIAN_NAME_MAPPING.update({'rest': 'librest'}) if apt_pkg.version_compare(version, '3.7') > 0: DEBIAN_NAME_MAPPING.update({'clutter-gst': 'clutter-gst-2.0'}) DEBIAN_NAME_MAPPING.update({'libgee': 'libgee-0.8'}) DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.20'}) if apt_pkg.version_compare(version, '3.9') > 0: DEBIAN_NAME_MAPPING.update({'gstreamer': 'gstreamer1.0'}) DEBIAN_NAME_MAPPING.update({'gst-plugins-base': 'gst-plugins-base1.0'}) DEBIAN_NAME_MAPPING.update({'gst-plugins-good': 'gst-plugins-good1.0'}) DEBIAN_NAME_MAPPING.update({'gst-plugins-bad': 'gst-plugins-bad1.0'}) DEBIAN_NAME_MAPPING.update({'gst-plugins-ugly': 'gst-plugins-ugly1.0'}) DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.22'}) if apt_pkg.version_compare(version, '3.11') > 0: DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.24'}) if apt_pkg.version_compare(version, '3.13') > 0: DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.26'}) DEBIAN_NAME_MAPPING.update({'vte': 'vte2.91'}) if apt_pkg.version_compare(version, '3.15') > 0: DEBIAN_NAME_MAPPING.update({'vala': 'vala-0.28'}) if apt_pkg.version_compare(version, '3.17') > 0: DEBIAN_NAME_MAPPING.update({'vala': 'vala'}) if __name__ == '__main__': if not os.path.exists(CACHE_DIR): os.mkdir(CACHE_DIR) parser = OptionParser() parser.add_option('--version', dest='version', metavar='VERSION', default='2.26') parser.add_option('--output', dest='output', metavar='OUTPUT', default='-') options, args = parser.parse_args() version = options.version amend_debian_name_mapping(version) if options.output == '-': output = sys.stdout else: output = StringIO() print >> output, HTML_AT_TOP % {'version': version} gnome_packages = get_gnome_packages(version) telepathy_packages = get_telepathy_packages() modules = gnome_packages + telepathy_packages Package._all_debian_package_versions = {} for i in range(0, len(modules), 10): sliced_modules = modules[i:i+10] url = 'https://api.ftp-master.debian.org/madison?f=json&S=true&package=%s' % urllib.quote(' '.join( [x.debian_name for x in sliced_modules])) #print url #Package._all_debian_package_versions.update(json.load(urllib2.urlopen(url, context=context))[0]) Package._all_debian_package_versions.update( json.loads(command_cache_execute(['curl', '-k', '--silent', url]))[0]) if not gnome_packages: print >> output, '' for suite in get_gnome_suites(version): print >> output, '' % suite suite_modules = [x for x in modules if x.suite == suite] suite_modules.sort(lambda x,y: cmp( (x.language, x.name.lower()), (y.language, y.name.lower()))) for module in suite_modules: module.print_tr(output) module.print_arch_details_tr(output) print >> output, '' % 'Telepathy' for module in telepathy_packages: module.print_tr(output) module.print_arch_details_tr(output) print >> output, '' print >> output, '
Upstream Debian Archs
%smissing
%s / %s%sPTS%sallall%s/%s
' for version, debsuite, archs in packages: print >> output, debsuite, 'has', version, 'for', ', '.join(archs) print >> output, '
' print >> output, '
Getting this script working on GNOME release days is hard; let\'s go shopping!
%s
%s
' total_modules = len(modules) ok_testing = len([x for x in modules if x.get_status_and_debian_version()[0] in ('ok-testing',)]) ok_unstable = len([x for x in modules if x.get_status_and_debian_version()[0] in ('ok', 'ok-testing',)]) print >> output, '

' print >> output, '%s out of %s packages are up-to-date in unstable (%d%%)' % ( ok_unstable, total_modules, 100.*ok_unstable/total_modules) print >> output, '
%s out of %s packages are up-to-date in testing (%d%%)' % ( ok_testing, total_modules, 100.*ok_testing/total_modules) print >> output, '

' print >> output, '
' print >> output, '' print >> output, '' if options.output != '-': file(options.output, 'w').write(output.getvalue())