Ticket #12486: patchbot-scripts-5.0.patch

new file patchbot/README.txt

@@ +1 @@
+The patchbot only needs a Sage install and is started with
+
+    python patchbot.py [options]
+
+Type --help for a list of options, though most configuration is done via an
+optional JSON config file. This is what is invoked by sage --patchbot [...]
+
+The server needs a Python with Flask and mongod installed.  Installing numpy
+and PIL will allow multi-colored blurbs.  Start a monitoring loop with
+
+    python run_server.py
+
+Currently, the server is set up to run on port 21100, communicating with
+a mongod instance running on 21001.

new file patchbot/db.py

@@ +1 @@
+import os
+
+# mongod --port=21000 --dbpath=data
+import pymongo, gridfs
+from pymongo import Connection
+mongo_port = 21001
+
+mongodb = Connection(port=mongo_port).buildbot
+tickets = mongodb.tickets
+tickets.ensure_index('id', unique=True)
+tickets.ensure_index('status')
+tickets.ensure_index('authors')
+tickets.ensure_index('participants')
+tickets.ensure_index('reports.base')
+tickets.ensure_index('reports.machine')
+tickets.ensure_index('reports.time')
+
+logs = gridfs.GridFS(mongodb, 'logs')
+
+def lookup_ticket(ticket_id):
+    return tickets.find_one({'id': ticket_id})
+
+def save_ticket(ticket_data):
+    old = lookup_ticket(ticket_data['id'])
+    if old:
+        old.update(ticket_data)
+        ticket_data = old
+    tickets.save(ticket_data)

new file patchbot/http_post_file.py

@@ +1 @@
+# http://code.activestate.com/recipes/146306-http-client-to-post-using-multipartform-data/
+
+import httplib, mimetypes, mimetools, urllib2
+
+def post_multipart(url, fields, files):
+    """
+    Post fields and files to an http host as multipart/form-data.
+    fields is a sequence of (name, value) elements for regular form fields.
+    files is a sequence of (name, filename, value) elements for data to be uploaded as files
+    Return the server's response page.
+    """
+    content_type, body = encode_multipart_formdata(fields, files)
+    headers = {'Content-Type': content_type,
+               'Content-Length': str(len(body))}
+    r = urllib2.Request(url, body, headers)
+    return urllib2.urlopen(r).read()
+
+def encode_multipart_formdata(fields, files):
+    """
+    fields is a sequence of (name, value) elements for regular form fields.
+    files is a sequence of (name, filename, value) elements for data to be uploaded as files
+    Return (content_type, body) ready for httplib.HTTP instance
+    """
+    BOUNDARY = mimetools.choose_boundary()
+    CRLF = '\r\n'
+    L = []
+    if isinstance(fields, dict):
+        fields = fields.items()
+    for (key, value) in fields:
+        L.append('--' + BOUNDARY)
+        L.append('Content-Disposition: form-data; name="%s"' % key)
+        L.append('')
+        L.append(value)
+    for (key, filename, value) in files:
+        L.append('--' + BOUNDARY)
+        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
+        L.append('Content-Type: %s' % get_content_type(filename))
+        L.append('')
+        L.append(value)
+    L.append('--' + BOUNDARY + '--')
+    L.append('')
+    body = CRLF.join(L)
+    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
+    return content_type, body
+
+def get_content_type(filename):
+    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

new file patchbot/patchbot.py

@@ +1 @@
+#!/usr/bin/env python
+
+####################################################################
+#
+# This is the main script for the patchbot. It pulls patches from
+# trac, applies them, and publishes the results of the tests to a
+# server running serve.py.  Configuration is primarily done via an
+# optional conf.txt file passed in as a command line argument.
+#
+#          Author: Robert Bradshaw <robertwb@gmail.com>
+#
+#               Copyright 2010-11 (C) Google, Inc.
+#
+#  Distributed under the terms of the GNU General Public License (GPL)
+#  as published by the Free Software Foundation; either version 2 of
+#  the License, or (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+####################################################################
+
+
+import signal
+import getpass, platform
+import random, re, os, shutil, sys, subprocess, time, traceback
+import bz2, urllib2, urllib, json
+from optparse import OptionParser
+
+from http_post_file import post_multipart
+
+from trac import scrape, pull_from_trac
+from util import now_str as datetime, parse_datetime, prune_pending, do_or_die, get_base, compare_version, current_reports
+
+def filter_on_authors(tickets, authors):
+    if authors is not None:
+        authors = set(authors)
+    for ticket in tickets:
+        if authors is None or set(ticket['authors']).issubset(authors):
+            yield ticket
+
+def contains_any(key, values):
+    clauses = [{'key': value} for value in values]
+    return {'$or': clauses}
+
+def no_unicode(s):
+    return s.encode('ascii', 'replace').replace(u'\ufffd', '?')
+
+def get_ticket(server, return_all=False, **conf):
+    query = "raw&status=open&todo"
+    if 'trusted_authors' in conf:
+        query += "&authors=" + urllib.quote_plus(no_unicode(':'.join(conf['trusted_authors'])), safe=':')
+    try:
+        handle = urllib2.urlopen(server + "/ticket/?" + query)
+        all = json.load(handle)
+        handle.close()
+    except:
+        traceback.print_exc()
+        return
+    if 'trusted_authors' in conf:
+        all = filter_on_authors(all, conf['trusted_authors'])
+    all = filter(lambda x: x[0], ((rate_ticket(t, **conf), t) for t in all))
+    all.sort()
+    if return_all:
+        return all
+    if all:
+        return all[-1]
+
+def lookup_ticket(server, id):
+    url = server + "/ticket/?" + urllib.urlencode({'raw': True, 'query': json.dumps({'id': id})})
+    res = json.load(urllib2.urlopen(url))
+    if res:
+        return res[0]
+    else:
+        return scrape(id)
+
+def compare_machines(a, b, machine_match=None):
+    if isinstance(a, dict) or isinstance(b, dict):
+        # old format, remove
+        return (1,)
+    else:
+        if machine_match is not None:
+            a = a[:machine_match]
+            b = b[:machine_match]
+        diff = [x != y for x, y in zip(a, b)]
+        if len(a) != len(b):
+            diff.append(1)
+        return diff
+
+def rate_ticket(ticket, **conf):
+    rating = 0
+    if ticket['spkgs']:
+        return # can't handle these yet
+    elif not ticket['patches']:
+        return # nothing to do
+    for dep in ticket['depends_on']:
+        if isinstance(dep, basestring) and '.' in dep:
+            if compare_version(conf['base'], dep) < 0:
+                # Depends on a newer version of Sage than we're running.
+                return None
+    for author in ticket['authors']:
+        if author not in conf['trusted_authors']:
+            return
+        rating += conf['bonus'].get(author, 0)
+    for participant in ticket['participants']:
+        rating += conf['bonus'].get(participant, 0) # doubled for authors
+    rating += len(ticket['participants'])
+    # TODO: remove condition
+    if 'component' in ticket:
+        rating += conf['bonus'].get(ticket['component'], 0)
+    rating += conf['bonus'].get(ticket['status'], 0)
+    rating += conf['bonus'].get(ticket['priority'], 0)
+    rating += conf['bonus'].get(str(ticket['id']), 0)
+    redundancy = (100,)
+    prune_pending(ticket)
+    if not ticket.get('retry'):
+        for reports in current_reports(ticket, base=conf['base']):
+            redundancy = min(redundancy, compare_machines(reports['machine'], conf['machine'], conf['machine_match']))
+    if not redundancy[-1]:
+        return # already did this one
+    return redundancy, rating, -int(ticket['id'])
+
+def report_ticket(server, ticket, status, base, machine, user, log, plugins=[]):
+    print ticket['id'], status
+    report = {
+        'status': status,
+        'patches': ticket['patches'],
+        'deps': ticket['depends_on'],
+        'spkgs': ticket['spkgs'],
+        'base': base,
+        'user': user,
+        'machine': machine,
+        'time': datetime(),
+        'plugins': plugins,
+    }
+    fields = {'report': json.dumps(report)}
+    if status != 'Pending':
+        files = [('log', 'log', bz2.compress(open(log).read()))]
+    else:
+        files = []
+    try:
+        print post_multipart("%s/report/%s" % (server, ticket['id']), fields, files)
+    except:
+        traceback.print_exc()
+
+class TimeOut(Exception):
+    pass
+
+def alarm_handler(signum, frame):
+    raise Alarm
+
+class Tee:
+    def __init__(self, filepath, time=False, timeout=60*60*24):
+        self.filepath = filepath
+        self.time = time
+        self.timeout = timeout
+
+    def __enter__(self):
+        self._saved = os.dup(sys.stdout.fileno()), os.dup(sys.stderr.fileno())
+        self.tee = subprocess.Popen(["tee", self.filepath], stdin=subprocess.PIPE)
+        os.dup2(self.tee.stdin.fileno(), sys.stdout.fileno())
+        os.dup2(self.tee.stdin.fileno(), sys.stderr.fileno())
+        if self.time:
+            print datetime()
+            self.start_time = time.time()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None:
+            traceback.print_exc()
+        if self.time:
+            print datetime()
+            print int(time.time() - self.start_time), "seconds"
+        self.tee.stdin.close()
+        time.sleep(1)
+        os.dup2(self._saved[0], sys.stdout.fileno())
+        os.dup2(self._saved[1], sys.stderr.fileno())
+        time.sleep(1)
+        try:
+            signal.signal(signal.SIGALRM, alarm_handler)
+            signal.alarm(self.timeout)
+            self.tee.wait()
+            signal.alarm(0)
+        except TimeOut:
+            traceback.print_exc()
+            raise
+        return False
+
+
+class Timer:
+    def __init__(self):
+        self._starts = {}
+        self._history = []
+        self.start()
+    def start(self, label=None):
+        self._last_activity = self._starts[label] = time.time()
+    def finish(self, label=None):
+        try:
+            elapsed = time.time() - self._starts[label]
+        except KeyError:
+            elapsed = time.time() - self._last_activity
+        self._last_activity = time.time()
+        self.print_time(label, elapsed)
+        self._history.append((label, elapsed))
+    def print_time(self, label, elapsed):
+        print label, '--', int(elapsed), 'seconds'
+    def print_all(self):
+        for label, elapsed in self._history:
+            self.print_time(label, elapsed)
+
+# The sage test scripts could really use some cleanup...
+all_test_dirs = ["doc/common", "doc/en", "doc/fr", "sage"]
+
+status = {
+    'started': 'ApplyFailed',
+    'applied': 'BuildFailed',
+    'built'  : 'TestsFailed',
+    'tested' : 'TestsPassed',
+    'failed_plugin' : 'PluginFailed',
+}
+
+def plugin_boundary(name, end=False):
+    if end:
+        name = 'end ' + name
+    return ' '.join(('='*10, name, '='*10))
+
+
+def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
+    base = get_base(sage_root)
+    if ticket is None:
+        ticket = get_ticket(base=base, server=server, **conf)
+    else:
+        ticket = None, scrape(int(ticket))
+    if not ticket:
+        print "No more tickets."
+        if random.random() < 0.01:
+            cleanup(sage_root, server)
+        time.sleep(conf['idle'])
+        return
+    rating, ticket = ticket
+    print "\n" * 2
+    print "=" * 30, ticket['id'], "=" * 30
+    print ticket['title']
+    print "score", rating
+    print "\n" * 2
+    log_dir = sage_root + "/logs"
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+    log = '%s/%s-log.txt' % (log_dir, ticket['id'])
+    report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], user=conf['user'], log=None)
+    plugins_results = []
+    try:
+        with Tee(log, time=True, timeout=conf['timeout']):
+            t = Timer()
+            start_time = time.time()
+
+            state = 'started'
+            os.environ['MAKE'] = "make -j%s" % conf['parallelism']
+            os.environ['SAGE_ROOT'] = sage_root
+            # TODO: Ensure that sage-main is pristine.
+            pull_from_trac(sage_root, ticket['id'], force=True)
+            t.finish("Apply")
+            state = 'applied'
+
+            do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
+            t.finish("Build")
+            state = 'built'
+
+            working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
+            # Only the ones on this ticket.
+            patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
+            kwds = {
+                "original_dir": "%s/devel/sage-0" % sage_root,
+                "patched_dir": working_dir,
+                "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
+            }
+            for name, plugin in conf['plugins']:
+                try:
+                    print plugin_boundary(name)
+                    plugin(ticket, **kwds)
+                    passed = True
+                except Exception:
+                    traceback.print_exc()
+                    passed = False
+                finally:
+                    t.finish(name)
+                    print plugin_boundary(name, end=True)
+                    plugins_results.append((name, passed))
+
+            test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
+            if conf['parallelism'] > 1:
+                test_cmd = "-tp %s" % conf['parallelism']
+            else:
+                test_cmd = "-t"
+            do_or_die("$SAGE_ROOT/sage %s -sagenb %s" % (test_cmd, ' '.join(test_dirs)))
+            #do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
+            #do_or_die('sage -testall')
+            t.finish("Tests")
+            state = 'tested'
+
+            if not all(passed for name, passed in plugins_results):
+                state = 'failed_plugin'
+
+            print
+            t.print_all()
+    except Exception:
+        traceback.print_exc()
+    report_ticket(server, ticket, status=status[state], base=base, machine=conf['machine'], user=conf['user'], log=log, plugins=plugins_results)
+    return status[state]
+
+def cleanup(sage_root, server):
+    print "Looking up closed tickets."
+    closed_list = urllib2.urlopen(server + "?status=closed").read()
+    closed = set(m.groups()[0] for m in re.finditer(r"/ticket/(\d+)/", closed_list))
+    for branch in os.listdir(os.path.join(sage_root, "devel")):
+        if branch[:5] == "sage-":
+            if branch[5:] in closed:
+                to_delete = os.path.join(sage_root, "devel", branch)
+                print "Deleting closed ticket:", to_delete
+                shutil.rmtree(to_delete)
+    print "Done cleaning up."
+
+def default_trusted_authors(server):
+    handle = urllib2.urlopen(server + "/trusted/")
+    try:
+        return json.load(handle).keys()
+    finally:
+        handle.close()
+
+def machine_data():
+    system, node, release, version, arch = os.uname()
+    if system.lower() == "linux":
+        dist_name, dist_version, dist_id = platform.linux_distribution()
+        if dist_name:
+            return [dist_name, dist_version, arch, release, node]
+    return [system, arch, release, node]
+
+def parse_time_of_day(s):
+    def parse_interval(ss):
+        ss = ss.strip()
+        if '-' in ss:
+            start, end = ss.split('-')
+            return float(start), float(end)
+        else:
+            return float(ss), float(ss) + 1
+    return [parse_interval(ss) for ss in s.split(',')]
+
+def check_time_of_day(hours):
+    from datetime import datetime
+    now = datetime.now()
+    hour = now.hour + now.minute / 60.
+    for start, end in parse_time_of_day(hours):
+        if start < end:
+            if start <= hour <= end:
+                return True
+        elif hour <= end or start <= hour:
+            return True
+    return False
+
+def get_conf(path, server, **overrides):
+    if path is None:
+        unicode_conf = {}
+    else:
+        unicode_conf = json.load(open(path))
+    # defaults
+    conf = {
+        "idle": 300,
+        "time_of_day": "0-0", # midnight-midnight
+        "parallelism": 3,
+        "timeout": 3 * 60 * 60,
+        "plugins": ["plugins.commit_messages",
+                    "plugins.coverage",
+                    "plugins.trailing_whitespace",
+#                    "plugins.docbuild"
+                    ],
+        "bonus": {},
+        "machine": machine_data(),
+        "machine_match": 3,
+        "user": getpass.getuser(),
+    }
+    default_bonus = {
+        "needs_review": 1000,
+        "positive_review": 500,
+        "blocker": 100,
+        "critical": 50,
+    }
+    for key, value in unicode_conf.items():
+        conf[str(key)] = value
+    for key, value in default_bonus.items():
+        if key not in conf['bonus']:
+            conf['bonus'][key] = value
+    conf.update(overrides)
+    if "trusted_authors" not in conf:
+        conf["trusted_authors"] = default_trusted_authors(server)
+
+    def locate_plugin(name):
+        ix = name.rindex('.')
+        module = name[:ix]
+        name = name[ix+1:]
+        plugin = getattr(__import__(module, fromlist=[name]), name)
+        assert callable(plugin)
+        return plugin
+    conf["plugins"] = [(name, locate_plugin(name)) for name in conf["plugins"]]
+    return conf
+
+def main(args):
+    global conf
+
+    # Most configuration is done in the config file, which is reread between
+    # each ticket for live configuration of the patchbot.
+    parser = OptionParser()
+    parser.add_option("--config", dest="config")
+    parser.add_option("--sage-root", dest="sage_root", default=os.environ.get('SAGE_ROOT'))
+    parser.add_option("--server", dest="server", default="http://patchbot.sagemath.org/")
+    parser.add_option("--count", dest="count", default=1000000)
+    parser.add_option("--ticket", dest="ticket", default=None)
+    parser.add_option("--list", dest="list", default=False)
+    parser.add_option("--skip-base", dest="skip_base", default=False)
+    (options, args) = parser.parse_args(args)
+
+    conf_path = options.config and os.path.abspath(options.config)
+    if options.ticket:
+        tickets = [int(t) for t in options.ticket.split(',')]
+        count = len(tickets)
+    else:
+        tickets = None
+        count = int(options.count)
+
+    conf = get_conf(conf_path, options.server)
+    if options.list:
+        for score, ticket in get_ticket(base=get_base(options.sage_root), server=options.server, return_all=True, **conf):
+            print score, ticket['id'], ticket['title']
+            print ticket
+            print
+        sys.exit(0)
+
+    print "WARNING: Assuming sage-main is pristine."
+    if options.sage_root == os.environ.get('SAGE_ROOT'):
+        print "WARNING: Do not use this copy of sage while the patchbot is running."
+
+    if not options.skip_base:
+        clean = lookup_ticket(options.server, 0)
+        def good(report):
+            return report['machine'] == conf['machine'] and report['status'] == 'TestsPassed'
+        if not any(good(report) for report in current_reports(clean, base=get_base(options.sage_root))):
+            res = test_a_ticket(ticket=0, sage_root=options.sage_root, server=options.server)
+            if res != 'TestsPassed':
+                print "\n\n"
+                while True:
+                    print "Failing tests in your install: %s. Continue anyways? [y/N] " % res
+                    ans = sys.stdin.readline().lower().strip()
+                    if ans == '' or ans[0] == 'n':
+                        sys.exit(1)
+                    elif ans[0] == 'y':
+                        break
+
+    for _ in range(count):
+        if tickets:
+            ticket = tickets.pop(0)
+        else:
+            ticket = None
+        conf = get_conf(conf_path, options.server)
+        if check_time_of_day(conf['time_of_day']):
+            test_a_ticket(ticket=ticket, sage_root=options.sage_root, server=options.server)
+        else:
+            print "Idle."
+            time.sleep(conf['idle'])
+
+if __name__ == '__main__':
+    # allow this script to serve as a single entry point for bots and the server
+    args = list(sys.argv)
+    if len(args) > 1 and args[1] == '--serve':
+        del args[1]
+        from serve import main
+    main(args)

new file patchbot/plugins.py

@@ +1 @@
+"""
+A plugin is any callable.
+
+It is called after the ticket has been successfully applied and built,
+but before tests are run. It should print out any analysis to stdout,
+raising an exception if anything went wrong.
+
+The parameters are as follows:
+
+   ticket -- a dictionary of all the ticket informaton
+   original_dir -- pristine sage-main directory
+   patched_dir -- patched sage-branch directory for this ticket
+   patchs -- a list of absolute paths to the patch files for this ticket
+
+It is recommended that a plugin ignore extra keywords to be
+compatible with future options.
+"""
+
+import re, os, sys
+
+from trac import do_or_die
+
+
+def coverage(ticket, **kwds):
+    do_or_die('$SAGE_ROOT/sage -coverageall')
+
+def docbuild(ticket, **kwds):
+    do_or_die('$SAGE_ROOT/sage -docbuild --jsmath reference html')
+
+def trailing_whitespace(ticket, patches, **kwds):
+    ignore_empty = True
+    bad_lines = 0
+    trailing = re.compile("\\+.*\\s+$")
+    for patch_path in patches:
+        patch = os.path.basename(patch_path)
+        print patch
+        for ix, line in enumerate(open(patch_path)):
+            line = line.strip("\n")
+            m = trailing.match(line)
+            if m:
+                print "    %s:%s %s$" % (patch, ix+1, line)
+                if line.strip() == '+' and ignore_empty:
+                    pass
+                else:
+                    bad_lines += 1
+    msg = "Trailing whitespace inserted on %s %slines." % (bad_lines, "non-empty " if ignore_empty else "")
+    print msg
+    if bad_lines > 0:
+        raise ValueError(msg)
+
+def commit_messages(ticket, patches, **kwds):
+    for patch_path in patches:
+        patch = os.path.basename(patch_path)
+        print "Looking at", patch
+        header = []
+        for line in open(patch_path):
+            if line.startswith('diff '):
+                break
+            header.append(line)
+        else:
+            print ''.join(header[:10])
+            raise ValueError("Not a valid patch file: " + patch)
+        print ''.join(header)
+        if header[0].strip() != "# HG changeset patch":
+            raise ValueError("Not a mercurial patch file: " + patch)
+        for line in header:
+            if not line.startswith('# '):
+                # First description line
+                if line.startswith('[mq]'):
+                    raise ValueError("Mercurial queue boilerplate")
+                elif not re.search(r"\b%s\b" % ticket['id'], line):
+                    print "Ticket number not in first line of comments: " + patch
+                break
+        else:
+            raise ValueError("No patch comments:" + patch)
+        print
+    print "All patches good."
+
+if __name__ == '__main__':
+    plugin = globals()[sys.argv[1]]
+    plugin(-1, patches=sys.argv[2:])

new file patchbot/run_server.py

@@ +1 @@
+#!/usr/bin/env python
+
+import os, signal, subprocess, sys, time, traceback, urllib2
+
+if not hasattr(subprocess.Popen, 'send_signal'):
+    def send_signal(self, sig):
+        os.kill(self.pid, sig)
+    subprocess.Popen.send_signal = send_signal
+
+DATABASE = "../data"
+
+# The server hangs while connecting to trac, so we poll it and
+# restart if needed.
+
+HTTP_TIMEOUT = 60
+POLL_INTERVAL = 180
+KILL_WAIT = 5
+
+p = None
+try:
+    # Start mongodb
+    mongo_process = subprocess.Popen(["mongod", "--port=21001", "--dbpath=" + DATABASE], stderr=subprocess.STDOUT)
+
+    # Run the server
+    while True:
+
+        if p is None or p.poll() is not None:
+            # The subprocess died.
+            restart = True
+        else:
+            try:
+                print "Testing url..."
+                urllib2.urlopen("http://patchbot.sagemath.org/", timeout=HTTP_TIMEOUT)
+                print "    ...good"
+                restart = False
+            except urllib2.URLError, e:
+                print "    ...bad", e
+                restart = True
+
+        if restart:
+            if p is not None and p.poll() is None:
+                print "SIGTERM"
+                p.send_signal(signal.SIGTERM)
+                time.sleep(KILL_WAIT)
+                if p.poll() is None:
+                    print "SIGKILL"
+                    p.kill()
+                    time.sleep(KILL_WAIT)
+
+            print "Starting server..."
+            base = open("base.txt").read().strip()
+            p = subprocess.Popen([sys.executable, "serve.py", "--base=" + base, "--port=21100"])
+            open("server.pid", "w").write(str(p.pid))
+            print "    ...done."
+        time.sleep(POLL_INTERVAL)
+
+finally:
+    traceback.print_exc()
+    mongo_process.send_signal(signal.SIGTERM)
+    if p is not None and p.poll() is None:
+        p.kill()
+

new file patchbot/serve.py

@@ +1 @@
+import os, sys, bz2, json, traceback, re, collections
+from cStringIO import StringIO
+from optparse import OptionParser
+from flask import Flask, render_template, make_response, request, Response
+import pymongo
+import trac
+import patchbot
+import db
+
+from db import tickets, logs
+from util import now_str, current_reports
+
+app = Flask(__name__)
+
+@app.route("/reports")
+def reports():
+    pass
+
+@app.route("/trusted")
+@app.route("/trusted/")
+def trusted_authors():
+    authors = collections.defaultdict(int)
+    for ticket in tickets.find({'status': 'closed : fixed'}):
+        for author in ticket["authors"]:
+            authors[author] += 1
+    if 'pretty' in request.args:
+        indent = 4
+    else:
+        indent = None
+    response = make_response(json.dumps(authors, default=lambda x: None, indent=indent))
+    response.headers['Content-type'] = 'text/plain'
+    return response
+
+@app.route("/")
+@app.route("/ticket")
+@app.route("/ticket/")
+def ticket_list():
+    authors = None
+    if 'query' in request.args:
+        query = json.loads(request.args.get('query'))
+    else:
+        status = request.args.get('status', 'needs_review')
+        if status == 'all':
+            query = {}
+        elif status in ('new', 'closed'):
+            query = {'status': {'$regex': status + '.*' }}
+        elif status in ('open'):
+            query = {'status': {'$regex': 'needs_.*|positive_review' }}
+        else:
+            query = {'status': status}
+        if 'todo' in request.args:
+            query['patches'] = {'$not': {'$size': 0}}
+            query['spkgs'] = {'$size': 0}
+        if 'authors' in request.args:
+            authors = request.args.get('authors').split(':')
+            query['authors'] = {'$in': authors}
+    if 'order' in request.args:
+        order = request.args.get('order')
+    else:
+        order = 'last_activity'
+    if 'base' in request.args:
+        base = request.args.get('base')
+        if base == 'all':
+            base = None
+    else:
+        base = global_base
+    if 'author' in request.args:
+        query['authors'] = request.args.get('author')
+    if 'participant' in request.args:
+        query['participants'] = request.args.get('participant')
+    all = patchbot.filter_on_authors(tickets.find(query).sort(order), authors)
+    if 'raw' in request.args:
+        if 'pretty' in request.args:
+            indent = 4
+        else:
+            indent = None
+        response = make_response(json.dumps(list(all), default=lambda x: None, indent=indent))
+        response.headers['Content-type'] = 'text/plain'
+        return response
+    summary = dict((key, 0) for key in status_order)
+    def preprocess(all):
+        for ticket in all:
+            ticket['report_count'], ticket['report_status'], ticket['report_status_composite'] = get_ticket_status(ticket, base)
+            if 'reports' in ticket:
+                ticket['pending'] = len([r for r in ticket['reports'] if r['status'] == 'Pending'])
+            summary[ticket['report_status']] += 1
+            yield ticket
+    ticket0 = db.lookup_ticket(0)
+    versions = list(set(report['base'] for report in ticket0['reports']))
+    versions.sort(trac.compare_version)
+    versions = [(v, get_ticket_status(ticket0, v)) for v in versions if v != '4.7.']
+    return render_template("ticket_list.html", tickets=preprocess(all), summary=summary, base=base, base_status=get_ticket_status(db.lookup_ticket(0), base), versions=versions, status_order=status_order)
+
+def format_patches(ticket, patches, deps=None, required=None):
+    if deps is None:
+        deps = []
+    if required is not None:
+        required = set(required)
+    def format_item(item):
+        if required is None or item in required:
+            note = ""
+        else:
+            note = "<span style='color: red'>(mismatch)</span>"
+        item = str(item)
+        if '#' in item:
+            url = trac.get_patch_url(ticket, item, raw=False)
+            title = item
+        elif '.' in item:
+            url = '/?base=%s' % item
+            title = 'sage-%s' % item
+        else:
+            url = '/ticket/%s' % item
+            title = '#%s' % item
+        return "<a href='%s'>%s</a> %s" % (url, title, note)
+
+    missing_deps = missing_patches = ''
+    if required is not None:
+        required_patches_count = len([p for p in required if '#' in str(p)])
+        if len(deps) < len(required) - required_patches_count:
+            missing_deps = "<li><span style='color: red'>(missing deps)</span>\n"
+        if len(patches) < required_patches_count:
+            missing_patches = "<li><span style='color: red'>(missing patches)</span>\n"
+    return ("<ol>"
+        + missing_deps
+        + "<li>\n"
+        + "\n<li>".join(format_item(patch) for patch in (deps + patches))
+        + missing_patches
+        + "</ol>")
+
+@app.route("/ticket/<int:ticket>/")
+def render_ticket(ticket):
+    try:
+        info = trac.scrape(ticket, db=db, force='force' in request.args)
+    except:
+        info = tickets.find_one({'id': ticket})
+    if info is None:
+        return "No such ticket."
+    if 'kick' in request.args:
+        info['retry'] = True
+        db.save_ticket(info)
+    if 'reports' in info:
+        info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
+    else:
+        info['reports'] = []
+
+    old_reports = list(info['reports'])
+    patchbot.prune_pending(info)
+    if old_reports != info['reports']:
+        db.save_ticket(info)
+
+    def format_info(info):
+        new_info = {}
+        for key, value in info.items():
+            if key == 'patches':
+                new_info['patches'] = format_patches(ticket, value)
+            elif key == 'reports' or key == 'pending':
+                pass
+            elif key == 'depends_on':
+                new_info[key] = ', '.join("<a href='/ticket/%s'>%s</a>" % (a, a) for a in value)
+            elif key == 'authors':
+                new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
+            elif key == 'participants':
+                new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
+            elif isinstance(value, list):
+                new_info[key] = ', '.join(value)
+            elif key not in ('id', '_id'):
+                new_info[key] = value
+        return new_info
+    def preprocess_reports(all):
+        for item in all:
+            if 'patches' in item:
+                required = info['depends_on'] + info['patches']
+                item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
+            if item['base'] != base:
+                item['base'] = "<span style='color: red'>%s</span>" % item['base']
+            if 'time' in item:
+                item['log'] = log_name(info['id'], item)
+            yield item
+    return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2])
+
+# The fact that this image is in the trac template lets the patchbot know
+# when a page gets updated.
+@app.route("/ticket/<int:ticket>/status.png")
+def render_ticket_status(ticket):
+    try:
+        info = trac.scrape(ticket, db=db)
+    except:
+        info = tickets.find_one({'id': ticket})
+    status = get_ticket_status(info, base=base)[2]
+    response = make_response(create_status_image(status))
+    response.headers['Content-type'] = 'image/png'
+    response.headers['Cache-Control'] = 'no-cache'
+    return response
+
+def get_or_set(ticket, key, default):
+    if key in ticket:
+        value = ticket[key]
+    else:
+        value = ticket[key] = default
+    return value
+
+@app.route("/report/<int:ticket_id>", methods=['POST'])
+def post_report(ticket_id):
+    try:
+        ticket = db.lookup_ticket(ticket_id)
+        if ticket is None:
+            ticket = trac.scrape(ticket_id)
+        if 'reports' not in ticket:
+            ticket['reports'] = []
+        report = json.loads(request.form.get('report'))
+        assert isinstance(report, dict)
+        for fld in ['status', 'patches', 'spkgs', 'base', 'machine', 'time']:
+            assert fld in report
+        patchbot.prune_pending(ticket, report['machine'])
+        ticket['reports'].append(report)
+        if report['status'] != 'Pending':
+            db.logs.put(request.files.get('log'), _id=log_name(ticket_id, report))
+        if 'retry' in ticket:
+            ticket['retry'] = False
+        ticket['last_activity'] = now_str()
+        db.save_ticket(ticket)
+        return "ok"
+    except:
+        traceback.print_exc()
+        return "error"
+
+def log_name(ticket_id, report):
+    return "/log/%s/%s/%s" % (ticket_id, '/'.join(report['machine']), report['time'])
+
+
+def shorten(lines):
+    timing = re.compile(r'\s*\[\d+\.\d* s\]\s*$')
+    skip = re.compile(r'(sage -t.*\(skipping\))|(byte-compiling)|(copying)|(\S+: \d+% \(\d+ of \d+\))$')
+    gcc = re.compile('(gcc)|(g\+\+)')
+    prev = None
+    for line in StringIO(lines):
+        if skip.match(line):
+            pass
+        elif prev is None:
+            prev = line
+        elif prev.startswith('sage -t') and timing.match(line):
+            prev = None
+        elif prev.startswith('python `which cython`') and '-->' in line:
+            prev = None
+        elif gcc.match(prev) and (gcc.match(line) or line.startswith('Time to execute')):
+            prev = line
+        else:
+            yield prev
+            prev = line
+
+    if prev is not None:
+        yield prev
+
+def extract_plugin_log(data, plugin):
+    from patchbot import plugin_boundary
+    start = plugin_boundary(plugin) + "\n"
+    end = plugin_boundary(plugin, end=True) + "\n"
+    all = []
+    include = False
+    for line in StringIO(data):
+        if line == start:
+            include = True
+        if include:
+            all.append(line)
+        if line == end:
+            break
+    return ''.join(all)
+
+@app.route("/ticket/<id>/log/<path:log>")
+def get_ticket_log(id, log):
+    return get_log(log)
+
+@app.route("/log/<path:log>")
+def get_log(log):
+    path = "/log/" + log
+    if not logs.exists(path):
+        data = "No such log!"
+    else:
+        data = bz2.decompress(logs.get(path).read())
+    if 'plugin' in request.args:
+        data = extract_plugin_log(data, request.args.get('plugin'))
+    if 'short' in request.args:
+        response = Response(shorten(data), direct_passthrough=True)
+    else:
+        response = make_response(data)
+    response.headers['Content-type'] = 'text/plain'
+    return response
+
+status_order = ['New', 'ApplyFailed', 'BuildFailed', 'TestsFailed', 'PluginFailed', 'TestsPassed', 'Pending', 'NoPatch', 'Spkg']
+# TODO: cleanup old records
+# status_order += ['started', 'applied', 'built', 'tested']
+
+status_colors = {
+    'New'        : 'white',
+    'ApplyFailed': 'red',
+    'BuildFailed': 'red',
+    'TestsFailed': 'yellow',
+    'TestsPassed': 'green',
+    'PluginFailed': 'blue',
+    'Pending'    : 'white',
+    'NoPatch'    : 'purple',
+    'Spkg'       : 'purple',
+}
+
+@app.route("/blob/<status>")
+def status_image(status):
+    response = make_response(create_status_image(status))
+    response.headers['Content-type'] = 'image/png'
+    response.headers['Cache-Control'] = 'max-age=3600'
+    return response
+
+def create_status_image(status):
+    if ',' in status:
+        status_list = status.split(',')
+        if len(set(status_list)) == 1:
+            status = status_list[0]
+        else:
+            try:
+                from PIL import Image
+                import numpy
+                path = 'images/_cache/' + ','.join(status_list) + '-blob.png'
+                if not os.path.exists(path):
+                    composite = numpy.asarray(Image.open(status_image(status_list[0]))).copy()
+                    height, width, _ = composite.shape
+                    for ix, status in enumerate(reversed(status_list)):
+                        slice = numpy.asarray(Image.open(status_image(status)))
+                        start = ix * width / len(status_list)
+                        end = (ix + 1) * width / len(status_list)
+                        composite[:,start:end,:] = slice[:,start:end,:]
+                    if not os.path.exists('images/_cache'):
+                        os.mkdir('images/_cache')
+                    Image.fromarray(composite, 'RGBA').save(path)
+                return open(path).read()
+            except ImportError:
+                print "here"
+                status = min_status(status_list)
+    return open(status_image(status)).read()
+
+def status_image(status):
+    return 'images/%s-blob.png' % status_colors[status]
+
+def min_status(status_list):
+    index = min(status_order.index(status) for status in status_list)
+    return status_order[index]
+
+@app.route("/robots.txt")
+def robots():
+    return """
+User-agent: *
+Disallow: /ticket/1303/status.png
+Disallow: /blob/
+Crawl-delay: 5
+    """.lstrip()
+
+@app.route("/favicon.ico")
+def robots():
+    response = make_response(open('images/%s-blob.png' % status_colors['TestsPassed']).read())
+    response.headers['Content-type'] = 'image/png'
+    return response
+
+def get_ticket_status(ticket, base=None):
+    all = current_reports(ticket, base=base)
+    if len(all):
+        status_list = [report['status'] for report in all]
+        if len(set(status_list)) == 1:
+            composite = single = status_list[0]
+        else:
+            composite = ','.join(status_list)
+            single = min_status(status_list)
+        return len(all), single, composite
+    elif ticket['spkgs']:
+        return 0, 'Spkg', 'Spkg'
+    elif not ticket['patches']:
+        return 0, 'NoPatch', 'NoPatch'
+    else:
+        return 0, 'New', 'New'
+
+def main(args):
+    parser = OptionParser()
+    parser.add_option("-b", "--base", dest="base")
+    parser.add_option("-p", "--port", dest="port")
+    parser.add_option("--debug", dest="debug", default=True)
+    (options, args) = parser.parse_args(args)
+
+    global global_base, base
+    global_base = base = options.base
+    app.run(debug=options.debug, host="0.0.0.0", port=int(options.port))
+
+if __name__ == '__main__':
+    main(sys.argv)

new file patchbot/templates/base.html

@@ +1 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+{% block head %}
+{% endblock head %}
+</head>
+<body>
+
+{% block body %}
+
+{% endblock body %}
+
+<script type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-20037535-1']);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</script>
+</body>
+</html>

new file patchbot/templates/ticket.html

@@ +1 @@
+{% extends 'base.html' %}
+
+{% block head %}
+<title>#{{ticket}} PatchBot Results</title>
+<link rel="shortcut icon" href="/blob/{{status}}" />
+{% endblock %}
+
+{% block body %}
+
+<h2>
+<img src="/blob/{{status}}">
+{{ticket}}
+</h2>
+<a href="http://trac.sagemath.org/sage_trac/ticket/{{ticket}}">
+{{info.title}}
+</a>
+<br><br>
+
+<table>
+{% for key, value in info.items(): %}
+<tr>
+<td align='right'>{{key}}:</td>
+<td>{{value|safe}}</td>
+</tr>
+{% endfor %}
+</table>
+<br>
+<hr>
+<table width='100%'>
+{% for report in reports: %}
+<tr>
+<td><img src="/blob/{{report.status}}"></td>
+<td><b>{{report.status}}</b></td>
+<td>{{report.base|safe}}</td>
+<td>{{'/'.join(report.machine)}}</td>
+<td>{{report.time}}</td>
+<td align='center'><a href='{{report.log}}'>log</a> <a href='{{report.log}}?short'>shortlog</a></td>
+</tr>
+<tr>
+<td colspan=2></td>
+<td colspan=3 valign='top'>
+{{report.patch_list|safe}}
+</td>
+<td valign='top'>
+<ul style='list-style-type: none'>
+{% for plugin, status in report.get('plugins', []) %}
+<li>
+<img height="16" src="/blob/{{['PluginFailed', 'TestsPassed'][status]}}">
+<a href='{{report.log}}?plugin={{plugin}}'>{{plugin}}</a>
+{% endfor %}
+</ul>
+</td>
+</tr>
+{% endfor %}
+</table>
+
+{% endblock %}

new file patchbot/templates/ticket_list.html

@@ +1 @@
+{% extends 'base.html' %}
+
+{% block head %}
+<title>Sage PatchBot</title>
+<link rel="shortcut icon" href="/blob/{{base_status[1]}}" />
+{% endblock %}
+
+{% block body %}
+
+<h1>Sage Patchbot</h1>
+<img src='/blob/{{base_status[1]}}' height=16>
+Results against Sage {{base}}
+(<a href='/ticket/0'>{{base_status[0]}} reports</a>).<br><br>
+See results against
+{% for version, status in versions: %}
+<img src='/blob/{{status[1]}}' height=16>
+<a href='?base={{version}}'>{{version}}</a>
+{% endfor %}
+
+<hr>
+
+<table>
+{% for ticket in tickets: %}
+<tr>
+<td>{{'*' if ticket.pending else ''}}</td>
+<td>{{ticket.report_count}}</td>
+<td><img src='/blob/{{ticket.report_status_composite}}' alt='{{ticket.report_status_composite}}' title='{{ticket.report_status_composite}}' height=16></td>
+<td align='right'>
+    <a href="/ticket/{{ticket.id}}/">{{ticket.id}}</a>
+</td>
+<td>{{ticket.title}}</td>
+<td>{{ticket.status}}</td>
+</tr>
+{% endfor %}
+</table>
+
+<hr>
+
+<table>
+{% for status in status_order %}
+<tr>
+<td><img src='/blob/{{status}}' height=16></td>
+<td>{{status}}</td>
+<td>{{summary[status]}}</td>
+</tr>
+{% endfor %}
+</table>
+
+{% endblock %}

new file patchbot/trac.py

@@ +1 @@
+TRAC_URL = "http://trac.sagemath.org/sage_trac"
+
+import re, hashlib, urllib2, os, sys, traceback, time, subprocess
+
+from util import do_or_die, extract_version, compare_version, get_base, now_str
+
+def digest(s):
+    """
+    Computes a cryptographic hash of the string s.
+    """
+    return hashlib.md5(s).hexdigest()
+
+def get_url(url):
+    """
+    Returns the contents of url as a string.
+    """
+    try:
+        url = url.replace(' ', '%20')
+        handle = urllib2.urlopen(url, timeout=5)
+        data = handle.read()
+        handle.close()
+        return data
+    except:
+        print url
+        raise
+
+def get_patch_url(ticket, patch, raw=True):
+    if raw:
+        return "%s/raw-attachment/ticket/%s/%s" % (TRAC_URL, ticket, patch)
+    else:
+        return "%s/attachment/ticket/%s/%s" % (TRAC_URL, ticket, patch)
+
+def get_patch(ticket, patch):
+    return get_url(get_patch_url(ticket, patch))
+
+def scrape(ticket_id, force=False, db=None):
+    """
+    Scrapes the trac page for ticket_id, updating the database if needed.
+    """
+    ticket_id = int(ticket_id)
+    if ticket_id == 0:
+        if db is not None:
+            db_info = db.lookup_ticket(ticket_id)
+            if db_info is not None:
+                return db_info
+        return {
+            'id'            : ticket_id,
+            'title'         : 'base',
+            'page_hash'     : '0',
+            'status'        : 'base',
+            'priority'      : 'base',
+            'component'     : 'base',
+            'depends_on'    : [],
+            'spkgs'         : [],
+            'patches'       : [],
+            'authors'       : [],
+            'participants'  : [],
+        }
+
+    rss = get_url("%s/ticket/%s?format=rss" % (TRAC_URL, ticket_id))
+    page_hash = digest(rss) # rss isn't as brittle
+    if db is not None:
+        # TODO: perhaps the db caching should be extracted outside of this function...
+        db_info = db.lookup_ticket(ticket_id)
+        if not force and db_info is not None and db_info['page_hash'] == page_hash:
+            return db_info
+    # TODO: Is there a better format that still has all the information?
+    html = get_url("%s/ticket/%s" % (TRAC_URL, ticket_id))
+    authors = set()
+    patches = []
+    for patch, who in extract_patches(rss):
+        authors.add(who)
+        patches.append(patch + "#" + digest(get_patch(ticket_id, patch)))
+    authors = list(authors)
+    data = {
+        'id'            : ticket_id,
+        'title'         : extract_title(rss),
+        'page_hash'     : page_hash,
+        'status'        : extract_status(html),
+        'milestone'     : extract_milestone(html),
+        'merged'        : extract_merged(html),
+        'priority'      : extract_priority(html),
+        'component'     : extract_component(html),
+        'depends_on'    : extract_depends_on(html),
+        'spkgs'         : extract_spkgs(html),
+        'patches'       : patches,
+        'authors'       : authors,
+        'participants'  : extract_participants(rss),
+        'last_activity' : now_str(),
+    }
+    if db is not None:
+        db.save_ticket(data)
+        db_info = db.lookup_ticket(ticket_id)
+        return db_info
+    else:
+        return data
+
+def extract_tag(sgml, tag):
+    """
+    Find the first occurance of the tag start (including attributes) and
+    return the contents of that tag (really, up until the next end tag
+    of that type).
+
+    Crude but fast.
+    """
+    tag_name = tag[1:-1]
+    if ' ' in tag_name:
+        tag_name = tag_name[:tag_name.index(' ')]
+    end = "</%s>" % tag_name
+    start_ix = sgml.find(tag)
+    if start_ix == -1:
+        return None
+    end_ix = sgml.find(end, start_ix)
+    if end_ix == -1:
+        return None
+    return sgml[start_ix + len(tag) : end_ix].strip()
+
+def extract_status(html):
+    """
+    Extracts the status of a ticket from the html page.
+    """
+    status = extract_tag(html, '<span class="status">')
+    if status is None:
+        return 'unknown'
+    status = status.strip('()')
+    status = status.replace('defect', '').replace('enhancement', '').strip()
+    return status
+
+def extract_priority(html):
+    """
+    Extracts any spkgs for a ticket from the html page.
+    """
+    return extract_tag(html, '<td headers="h_priority">')
+
+def extract_milestone(html):
+    milestone_field = extract_tag(html, '<td headers="h_milestone">')
+    return extract_version(milestone_field)
+
+def extract_merged(html):
+    merged_field = extract_tag(html, '<td headers="h_merged">')
+    return extract_version(merged_field)
+
+def extract_component(html):
+    return extract_tag(html, '<td headers="h_component">')
+
+def extract_title(rss):
+    title = extract_tag(rss, '<title>')
+    return re.sub(r'.*#\d+:', '', title).strip()
+
+folded_regex = re.compile('all.*(folded|combined|merged)')
+subsequent_regex = re.compile('second|third|fourth|next|on top|after')
+attachment_regex = re.compile(r"<strong>attachment</strong>\s*set to <em>(.*)</em>", re.M)
+rebased_regex = re.compile(r"([-.]?rebased?)|(-v\d)")
+def extract_patches(rss):
+    """
+    Extracts the list of patches for a ticket from the rss feed.
+
+    Tries to deduce the subset of attached patches to apply based on
+
+        (1) "Apply ..." in comment text
+        (2) Mercurial .N naming
+        (3) "rebased" in name
+        (3) Chronology
+    """
+    all_patches = []
+    patches = []
+    authors = {}
+    for item in rss.split('<item>'):
+        who = extract_tag(item, '<dc:creator>')
+        description = extract_tag(item, '<description>').replace('&lt;', '<').replace('&gt;', '>')
+        m = attachment_regex.search(description)
+        comments = description[description.find('</ul>') + 1:]
+        # Look for apply... followed by patch names
+        for line in comments.lower().split('\n'):
+            if 'apply' in line:
+                new_patches = []
+                for p in line[line.index('apply'):].split(','):
+                    for pp in p.strip().split():
+                        if pp in all_patches:
+                            new_patches.append(pp)
+                if new_patches or (m and not subsequent_regex.search(line)):
+                    patches = new_patches
+            elif m and folded_regex.search(line):
+                patches = [] # will add this patch below
+        if m is not None:
+            attachment = m.group(1)
+            base, ext = os.path.splitext(attachment)
+            if '.' in base:
+                try:
+                    base2, ext2 = os.path.splitext(base)
+                    count = int(ext2[1:])
+                    for i in range(count):
+                        if i:
+                            older = "%s.%s%s" % (base2, i, ext)
+                        else:
+                            older = "%s%s" % (base2, ext)
+                        if older in patches:
+                            patches.remove(older)
+                except:
+                    pass
+            if rebased_regex.search(attachment):
+                older = rebased_regex.sub('', attachment)
+                if older in patches:
+                    patches.remove(older)
+            if ext in ('.patch', '.diff'):
+                all_patches.append(attachment)
+                patches.append(attachment)
+                authors[attachment] = who
+    return [(p, authors[p]) for p in patches]
+
+participant_regex = re.compile("<strong>attachment</strong>\w*set to <em>(.*)</em>")
+def extract_participants(rss):
+    """
+    Extracts any spkgs for a ticket from the html page.
+    """
+    all = set()
+    for item in rss.split('<item>'):
+        who = extract_tag(item, '<dc:creator>')
+        if who:
+            all.add(who)
+    return list(all)
+
+spkg_url_regex = re.compile(r"(?:(?:http://)|(?:/attachment/)).*?\.spkg")
+#spkg_url_regex = re.compile(r"http://.*?\.spkg")
+def extract_spkgs(html):
+    """
+    Extracts any spkgs for a ticket from the html page.
+
+    Just searches for urls ending in .spkg.
+    """
+    return list(set(spkg_url_regex.findall(html)))
+
+def min_non_neg(*rest):
+    non_neg = [a for a in rest if a >= 0]
+    if len(non_neg) == 0:
+        return rest[0]
+    elif len(non_neg) == 1:
+        return non_neg[0]
+    else:
+        return min(*non_neg)
+
+ticket_url_regex = re.compile(r"%s/ticket/(\d+)" % TRAC_URL)
+def extract_depends_on(html):
+    deps_field = extract_tag(html, '<td headers="h_dependencies">')
+    deps = []
+    for dep in re.finditer(r'ticket/(\d+)', deps_field):
+        deps.append(int(dep.group(1)))
+    version = re.search(r'sage-\d+(\.\d)+(\.\w+)?', deps_field)
+    if version:
+        deps.insert(0, version.group(0))
+    return deps
+
+
+
+safe = re.compile('[-+A-Za-z0-9._]*')
+def ensure_safe(items):
+    """
+    Raise an error if item has any spaces in it.
+    """
+    if isinstance(items, (str, unicode)):
+        m = safe.match(items)
+        if m is None or m.end() != len(items):
+            raise ValueError, "Unsafe patch name '%s'" % items
+    else:
+        for item in items:
+            ensure_safe(item)
+
+
+def pull_from_trac(sage_root, ticket, branch=None, force=None, interactive=None):
+    # Should we set/unset SAGE_ROOT and SAGE_BRANCH here? Fork first?
+    if branch is None:
+        branch = str(ticket)
+    if not os.path.exists('%s/devel/sage-%s' % (sage_root, branch)):
+        do_or_die('%s/sage -b main' % (sage_root,))
+        do_or_die('%s/sage -clone %s' % (sage_root, branch))
+    os.chdir('%s/devel/sage-%s' % (sage_root, branch))
+    if interactive:
+        raise NotImplementedError
+    if not os.path.exists('.hg/patches'):
+        do_or_die('hg qinit')
+        series = []
+    elif not os.path.exists('.hg/patches/series'):
+        series = []
+    else:
+        series = open('.hg/patches/series').read().split('\n')
+
+    base = get_base(sage_root)
+    desired_series = []
+    seen_deps = []
+    def append_patch_list(ticket, dependency=False):
+        if ticket in seen_deps:
+            return
+        print "Looking at #%s" % ticket
+        seen_deps.append(ticket)
+        data = scrape(ticket)
+        if dependency and 'closed' in data['status']:
+            merged = data.get('merged')
+            if merged is None:
+                merged = data.get('milestone')
+            if merged is None or compare_version(merged, base) <= 0:
+                print "#%s already applied (%s <= %s)" % (ticket, merged, base)
+                return
+        if data['spkgs']:
+            raise NotImplementedError, "Spkgs not yet handled."
+        if data['depends_on']:
+            for dep in data['depends_on']:
+                if isinstance(dep, basestring) and '.' in dep:
+                    if compare_version(base, dep) < 0:
+                        raise ValueError, "%s < %s for %s" % (base, dep, ticket)
+                    continue
+                append_patch_list(dep, dependency=True)
+        print "Patches for #%s:" % ticket
+        print "    " + "\n    ".join(data['patches'])
+        for patch in data['patches']:
+            patchfile, hash = patch.split('#')
+            desired_series.append((hash, patchfile, get_patch_url(ticket, patchfile)))
+    append_patch_list(ticket)
+
+    ensure_safe(series)
+    ensure_safe(patch for hash, patch, url in desired_series)
+
+    last_good_patch = '-a'
+    to_push = list(desired_series)
+    for series_patch, (hash, patch, url) in zip(series, desired_series):
+        if not series_patch:
+            break
+        next_hash = digest(open('.hg/patches/%s' % series_patch).read())
+#        print next_hash, hash, series_patch
+        if next_hash == hash:
+            to_push.pop(0)
+            last_good_patch = series_patch
+        else:
+            break
+
+    try:
+        if last_good_patch != '-a':
+            # In case it's not yet pushed...
+            if last_good_patch not in os.popen2('hg qapplied')[1].read().split('\n'):
+                do_or_die('hg qpush %s' % last_good_patch)
+        do_or_die('hg qpop %s' % last_good_patch)
+        for hash, patch, url in to_push:
+            if patch in series:
+                if not force:
+                    raise Exception, "Duplicate patch: %s" % patch
+                old_patch = patch
+                while old_patch in series:
+                    old_patch += '-old'
+                do_or_die('hg qrename %s %s' % (patch, old_patch))
+            do_or_die('hg qimport %s && hg qpush' % url)
+        do_or_die('hg qapplied')
+    except:
+        os.system('hg qpop -a')
+        raise
+
+
+def push_from_trac(sage_root, ticket, branch=None, force=None, interactive=None):
+    raise NotImplementedError
+
+
+
+if __name__ == '__main__':
+    force = False
+    apply = False
+    for ticket in sys.argv[1:]:
+        if ticket == '-f':
+            force = True
+            continue
+        if ticket == '-a':
+            apply = True
+            continue
+        if '-' in ticket:
+            start, end = ticket.split('-')
+            tickets = range(int(start), int(end) + 1)
+        else:
+            tickets = [int(ticket)]
+        for ticket in tickets:
+            try:
+                print ticket, scrape(ticket, force=force)
+                if apply:
+                    pull_from_trac(os.environ['SAGE_ROOT'], ticket, force=True)
+                time.sleep(1)
+            except Exception:
+                print "Error for", ticket
+                traceback.print_exc()
+        force = apply = False
+#    pull_from_trac('/Users/robertwb/sage/current', ticket, force=True)

new file patchbot/util.py

@@ +1 @@
+import os, re, subprocess, time
+
+
+DATE_FORMAT = '%Y-%m-%d %H:%M:%S %z'
+def now_str():
+    return time.strftime(DATE_FORMAT)
+
+def parse_datetime(s):
+    # The one thing Python can't do is parse dates...
+    return time.mktime(time.strptime(s[:-5].strip(), DATE_FORMAT[:-3])) + 60*int(s[-5:].strip())
+
+def prune_pending(ticket, machine=None, timeout=6*60*60):
+    if 'reports' in ticket:
+        reports = ticket['reports']
+    else:
+        return []
+    # TODO: is there a better way to handle time zones?
+    now = time.time() + 60 * int(time.strftime('%z'))
+    for report in list(reports):
+        if report['status'] == 'Pending':
+            t = parse_datetime(report['time'])
+            if report['machine'] == machine:
+                reports.remove(report)
+            elif now - t > timeout:
+                reports.remove(report)
+    return reports
+
+def current_reports(ticket, base=None, unique=False):
+    if 'reports' not in ticket:
+        return []
+    if unique:
+        seen = set()
+        def first(x):
+            if x in seen:
+                return False
+            else:
+                seen.add(x)
+                return True
+    else:
+        first = lambda x: True
+    reports = list(ticket['reports'])
+    reports.sort(lambda a, b: cmp(b['time'], a['time']))
+    return filter(lambda report: (ticket['patches'] == report['patches'] and
+                                  ticket['spkgs'] == report['spkgs'] and
+                                  ticket['depends_on'] == (report.get('deps') or []) and
+                                  (not base or base == report['base'])) and
+                                  first('/'.join(report['machine'])),
+                      reports)
+
+def do_or_die(cmd):
+    print cmd
+    res = os.system(cmd)
+    if res:
+        raise Exception, "%s %s" % (res, cmd)
+
+def extract_version(s):
+    m = re.search(r'\d+(\.\d+)+(\.\w+)', s)
+    if m:
+        return m.group(0)
+
+def compare_version(a, b):
+    a = a.replace("beta", "beta.")
+    b = b.replace("beta", "beta.")
+    a += '.z'
+    b += '.z'
+    def maybe_int(s):
+        try:
+            return 1, int(s)
+        except ValueError:
+            return 0, s
+    return cmp([maybe_int(v) for v in a.split('.')],
+               [maybe_int(v) for v in b.split('.')])
+
+def get_base(sage_root):
+    p = subprocess.Popen([os.path.join(sage_root, 'sage'), '-v'], stdout=subprocess.PIPE)
+    if p.wait():
+        raise ValueError, "Invalid sage_root='%s'" % sage_root
+    version_info = p.stdout.read()
+    return re.search(r'Sage Version ([\d.]+\w*)', version_info).groups()[0]