From 9773844bbd624e66949ac0e200c5cb0ac4386d81 Mon Sep 17 00:00:00 2001 From: Priit Laes Date: Thu, 5 Aug 2010 09:54:26 +0300 Subject: Added version check plugin --- grumpy/vdb/__init__.py | 0 grumpy/vdb/pypi.py | 45 +++++++++++++++++++++ utils/version_check.py | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 grumpy/vdb/__init__.py create mode 100644 grumpy/vdb/pypi.py create mode 100644 utils/version_check.py diff --git a/grumpy/vdb/__init__.py b/grumpy/vdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/grumpy/vdb/pypi.py b/grumpy/vdb/pypi.py new file mode 100644 index 0000000..2e27059 --- /dev/null +++ b/grumpy/vdb/pypi.py @@ -0,0 +1,45 @@ +from lxml.html import fromstring +import urllib + +class PyPi(object): + # TODO: Shall we use some kind of yaml or json based format instead? + # Or move this data into DB? + pkgs = { + # Package in Portage : Package in PyPi + 'dev-python/flask' : 'Flask', + 'dev-python/flask-openid' : 'Flask-OpenID', + 'dev-python/flask-sqlalchemy' : 'Flask-SQLAlchemy', + 'dev-python/mako' : 'Mako', + 'dev-python/sphinx' : 'Sphinx', + 'dev-python/virtualenv' : 'virtualenv', + 'dev-python/vobject' : 'vobject', + 'dev-python/werkzeug' : 'Werkzeug', + 'dev-python/xlwt' : 'xlwt', + 'dev-python/yolk' : 'yolk', + 'dev-python/yolk-portage' : 'yolk-portage', + } + + # Url for fetching version information + url = 'http://pypi.python.org/pypi?:action=index' + + def __init__(self): + pass + + def fetch_and_parse_all(self): + """Download and parse package version information.""" + + items = {} + #f = urllib.urlopen(self.url) + f = open('utils/pypi.html') + if f: + data = fromstring(f.read()).cssselect('table.list')[0] + for row in data.getchildren(): + val = row[0].getchildren() + if len(val) > 0: + pkg = val[0].text.encode('utf-8').split('\xc2\xa0') + if len(pkg) == 2: + if pkg[0] not in items.keys(): + items[pkg[0]] = [] + items[pkg[0]].append(pkg[1]) + f.close() + return items diff --git a/utils/version_check.py b/utils/version_check.py new file mode 100644 index 0000000..20d1caa --- /dev/null +++ b/utils/version_check.py @@ -0,0 +1,105 @@ +#! /usr/bin/env python +import os, sys +from datetime import datetime +from lxml.html import fromstring +from optparse import OptionParser + +path = os.path.join(os.path.dirname(__file__), os.path.pardir) +sys.path.insert(0, path) +del path + +from grumpy import app +from grumpy.models import db, Package, PkgIssue, Setting +from grumpy.utils import compare_version + +PLUGIN_NAME='ver_bumper::' + +def gc_collect(timestamp): + """Remove old QA issues from database returning number of rows deleted.""" + db.session.expire_all() + print ("DEBUG: Deleted %d old issues." % PkgIssue.query \ + .filter_by(plugin=PLUGIN_NAME) \ + .filter(PkgIssue.created_on < timestamp).delete(False)) + +def insert_issues(invalid): + """Insert QA issues into db.""" + if 'maintainer-needed@gentoo.org' in invalid: + h = Developer.query.filter_by(email='maintainer-needed@gentoo.org').first() + for pkg in h.packages: + pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'maintainer-needed')) + invalid.remove('maintainer-needed@gentoo.org') + for dev in invalid: + d = Developer.query.filter_by(email=dev).first() + for pkg in d.packages: + pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'unknown-dev', \ + 'Maintainer %s listed in metadata.xml is not as ' \ + 'an active Gentoo developer.' % dev)) + db.session.commit() + +if __name__ == '__main__': + from grumpy.vdb.pypi import PyPi + parser = OptionParser(usage="usage: %prog [options] CONFFILE") + (opts, args) = parser.parse_args() + if len(args) != 1: + parser.error("provide path to configuration file as first argument") + sys.exit(1) + # ... + pypi = PyPi() + items = pypi.fetch_and_parse_all() + # Gather package versions + updates = {} + missing = [] + for p in pypi.pkgs: + pyp = pypi.pkgs[p] + if pyp not in items.keys(): + missing.append([p, pyp]) + else: + updates[p] = items[pyp] + PLUGIN_NAME = PLUGIN_NAME + pypi.__class__.__name__ + with app.test_request_context(): + timestamp = datetime.now() + app.config.from_pyfile(args[0]) + try: + # Delete existing issues + print ("DEBUG: Deleted %d old issues." % PkgIssue.query \ + .filter_by(plugin=PLUGIN_NAME) \ + .filter(PkgIssue.created_on < timestamp).delete(False)) + db.session.commit() + + # File 'missing' package mapping issues + for item in missing: + p = Package.query.filter_by(key=item).first() + if not p: + print "DEBUG: package '%s' is missing from upstream." % item + continue + p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \ + 'missing-upstream', \ + 'No package "%s" found in upstream: %s' % \ + (item, pypi.__class__.__name__))) + # Run version checks + # TODO: Figure out slotted ebuilds + for pkg, ver in updates.iteritems(): + p = Package.query.filter_by(key=pkg).first() + if not p: + print "DEBUG: package '%s' is missing from portage." % item + continue + # We need to sort pkg versions + ver = sorted(ver, compare_version, reverse=True) + res = compare_version(ver[0], p.versions[0]) + if res == -1: + p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \ + 'version-mismatch', 'Package "%s" has newer version in ' \ + 'Portage than on "%s" upstream' % \ + (pkg, pypi.__class__.__name__))) + elif res == 1: + p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \ + 'version-bump', 'Package "%s" has newer version "%s" ' + 'available in "%s" upstream ' % \ + (pkg, p.versions[0], pypi.__class__.__name__))) + db.session.commit() + # Update settings and add info about last run.. + Setting.query.filter_by(name=PLUGIN_NAME).delete(False) + db.session.add(Setting(PLUGIN_NAME, str(timestamp))) + db.session.commit() + except Exception: + db.session.rollback() -- cgit v1.2.3-65-gdbad