#!/usr/bin/python import argparse import json import operator import re import sys import subprocess import time import urllib import urllib2 from datetime import datetime from xml.dom.minidom import parseString as parseXmlString from util import submit_commits from util import text_content from util import load_server_config def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--os-config-json', required=True, help='The path to a JSON that specifies how to fetch OS build information') parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard') parser.add_argument('--seconds-to-sleep', type=float, default=43200, help='The seconds to sleep between iterations') args = parser.parse_args() with open(args.os_config_json) as os_config_json: os_config_list = json.load(os_config_json) fetchers = [OSBuildFetcher(os_config) for os_config in os_config_list] while True: server_config = load_server_config(args.server_config_json) for fetcher in fetchers: fetcher.fetch_and_report_new_builds(server_config) print "Sleeping for %d seconds" % args.seconds_to_sleep time.sleep(args.seconds_to_sleep) # FIXME: Move other static functions into this class. class OSBuildFetcher: def __init__(self, os_config): self._os_config = os_config self._reported_revisions = set() def _fetch_available_builds(self): config = self._os_config repository_name = self._os_config['name'] if 'customCommands' in config: available_builds = [] for command in config['customCommands']: print "Executing", ' '.join(command['command']) available_builds += available_builds_from_command(repository_name, command['command'], command['linesToIgnore']) else: url = config['buildSourceURL'] print "Fetching available builds from", url available_builds = fetch_available_builds(repository_name, url, config['trainVersionMap']) return available_builds def fetch_and_report_new_builds(self, server_config): available_builds = self._fetch_available_builds() reported_revisions = self._reported_revisions print 'Found %d builds' % len(available_builds) available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds) self._assign_order(available_builds) print "Submitting %d builds" % len(available_builds) submit_commits(available_builds, server_config['server']['url'], server_config['worker']['name'], server_config['worker']['password']) reported_revisions |= set(map(lambda commit: commit['revision'], available_builds)) @staticmethod def _assign_order(builds): build_name_regex = re.compile(r'(?P\d+)(?P\w)(?P\d+)(?P\w*)') for commit in builds: match = build_name_regex.search(commit['revision']) major = int(match.group('major')) kind = ord(match.group('kind').upper()) - ord('A') minor = int(match.group('minor')) variant = ord(match.group('variant').upper()) - ord('A') + 1 if match.group('variant') else 0 commit['order'] = ((major * 100 + kind) * 10000 + minor) * 100 + variant def available_builds_from_command(repository_name, command, lines_to_ignore): try: output = subprocess.check_output(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: print "Failed:", str(error) return [] regex = re.compile(lines_to_ignore) return [{'repository': repository_name, 'revision': line} for line in output.split('\n') if not regex.match(line)] def fetch_available_builds(repository_name, url, train_version_map): output = urllib2.urlopen(url).read() try: xml = parseXmlString(output) except Exception, error: raise Exception(error, output) available_builds = [] for image in xml.getElementsByTagName('baseASR'): id = text_content(image.getElementsByTagName('id')[0]) train = text_content(image.getElementsByTagName('train')[0]) build = text_content(image.getElementsByTagName('build')[0]) if train not in train_version_map: continue available_builds.append({ 'repository': repository_name, 'revision': train_version_map[train] + ' ' + build}) return available_builds if __name__ == "__main__": main(sys.argv)