118 lines
4.5 KiB
Python
Executable File
118 lines
4.5 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
import argparse
|
|
import json
|
|
import operator
|
|
import re
|
|
import sys
|
|
import subprocess
|
|
import time
|
|
import urllib
|
|
import urllib2
|
|
|
|
from datetime import datetime
|
|
from xml.dom.minidom import parseString as parseXmlString
|
|
from util import submit_commits
|
|
from util import text_content
|
|
from util import load_server_config
|
|
|
|
|
|
def main(argv):
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--os-config-json', required=True, help='The path to a JSON that specifies how to fetch OS build information')
|
|
parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard')
|
|
parser.add_argument('--seconds-to-sleep', type=float, default=43200, help='The seconds to sleep between iterations')
|
|
args = parser.parse_args()
|
|
|
|
with open(args.os_config_json) as os_config_json:
|
|
os_config_list = json.load(os_config_json)
|
|
|
|
fetchers = [OSBuildFetcher(os_config) for os_config in os_config_list]
|
|
|
|
while True:
|
|
server_config = load_server_config(args.server_config_json)
|
|
for fetcher in fetchers:
|
|
fetcher.fetch_and_report_new_builds(server_config)
|
|
print "Sleeping for %d seconds" % args.seconds_to_sleep
|
|
time.sleep(args.seconds_to_sleep)
|
|
|
|
|
|
# FIXME: Move other static functions into this class.
|
|
class OSBuildFetcher:
|
|
def __init__(self, os_config):
|
|
self._os_config = os_config
|
|
self._reported_revisions = set()
|
|
|
|
def _fetch_available_builds(self):
|
|
config = self._os_config
|
|
repository_name = self._os_config['name']
|
|
|
|
if 'customCommands' in config:
|
|
available_builds = []
|
|
for command in config['customCommands']:
|
|
print "Executing", ' '.join(command['command'])
|
|
available_builds += available_builds_from_command(repository_name, command['command'], command['linesToIgnore'])
|
|
else:
|
|
url = config['buildSourceURL']
|
|
print "Fetching available builds from", url
|
|
available_builds = fetch_available_builds(repository_name, url, config['trainVersionMap'])
|
|
return available_builds
|
|
|
|
def fetch_and_report_new_builds(self, server_config):
|
|
available_builds = self._fetch_available_builds()
|
|
reported_revisions = self._reported_revisions
|
|
print 'Found %d builds' % len(available_builds)
|
|
|
|
available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
|
|
self._assign_order(available_builds)
|
|
|
|
print "Submitting %d builds" % len(available_builds)
|
|
submit_commits(available_builds, server_config['server']['url'], server_config['worker']['name'], server_config['worker']['password'])
|
|
reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
|
|
|
|
@staticmethod
|
|
def _assign_order(builds):
|
|
build_name_regex = re.compile(r'(?P<major>\d+)(?P<kind>\w)(?P<minor>\d+)(?P<variant>\w*)')
|
|
for commit in builds:
|
|
match = build_name_regex.search(commit['revision'])
|
|
major = int(match.group('major'))
|
|
kind = ord(match.group('kind').upper()) - ord('A')
|
|
minor = int(match.group('minor'))
|
|
variant = ord(match.group('variant').upper()) - ord('A') + 1 if match.group('variant') else 0
|
|
commit['order'] = ((major * 100 + kind) * 10000 + minor) * 100 + variant
|
|
|
|
|
|
def available_builds_from_command(repository_name, command, lines_to_ignore):
|
|
try:
|
|
output = subprocess.check_output(command, stderr=subprocess.STDOUT)
|
|
except subprocess.CalledProcessError as error:
|
|
print "Failed:", str(error)
|
|
return []
|
|
|
|
regex = re.compile(lines_to_ignore)
|
|
return [{'repository': repository_name, 'revision': line} for line in output.split('\n') if not regex.match(line)]
|
|
|
|
|
|
def fetch_available_builds(repository_name, url, train_version_map):
|
|
output = urllib2.urlopen(url).read()
|
|
try:
|
|
xml = parseXmlString(output)
|
|
except Exception, error:
|
|
raise Exception(error, output)
|
|
available_builds = []
|
|
for image in xml.getElementsByTagName('baseASR'):
|
|
id = text_content(image.getElementsByTagName('id')[0])
|
|
train = text_content(image.getElementsByTagName('train')[0])
|
|
build = text_content(image.getElementsByTagName('build')[0])
|
|
if train not in train_version_map:
|
|
continue
|
|
available_builds.append({
|
|
'repository': repository_name,
|
|
'revision': train_version_map[train] + ' ' + build})
|
|
|
|
return available_builds
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv)
|