haikuwebkit/Websites/perf.webkit.org/tools/sync-commits.py

#!/usr/bin/python

import argparse
import json
import os.path
import re
import subprocess
import sys
import time
import urllib2

from datetime import datetime
from abc import ABCMeta, abstractmethod
from xml.dom.minidom import parseString as parseXmlString
from util import load_server_config
from util import submit_commits
from util import text_content

# There are some buggy commit messages:
# Canonical link: https://commits.webkit.org/https://commits.webkit.org/232477@main
REVISION_IDENTIFIER_RE = re.compile(r'Canonical link: (https\://commits\.webkit\.org/)+(?P<revision_identifier>\d+@[\w\.\-]+)\n')

def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--repository-config-json', required=True, help='The path to a JSON file that specifies subversion syncing options')
    parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard')
    parser.add_argument('--seconds-to-sleep', type=float, default=900, help='The seconds to sleep between iterations')
    parser.add_argument('--max-fetch-count', type=int, default=10, help='The number of commits to fetch at once')
    parser.add_argument('--max-ancestor-fetch-count', type=int, default=100, help='The number of commits to fetch at once if some commits are missing previous commits')
    args = parser.parse_args()

    with open(args.repository_config_json) as repository_config_json:
        repositories = [load_repository(repository_info) for repository_info in json.load(repository_config_json)]

    while True:
        server_config = load_server_config(args.server_config_json)
        for repository in repositories:
            try:
                repository.fetch_commits_and_submit(server_config, args.max_fetch_count, args.max_ancestor_fetch_count)
            except Exception as error:
                print "Failed to fetch and sync:", error

        print "Sleeping for %d seconds..." % args.seconds_to_sleep
        time.sleep(args.seconds_to_sleep)


def load_repository(repository):
    if 'gitCheckout' in repository:
        return GitRepository(
            name=repository['name'], git_url=repository['url'], git_checkout=repository['gitCheckout'],
            git_branch=repository.get('branch'), report_revision_identifier_in_commit_msg=repository.get('reportRevisionIdentifier'),
            report_svn_revison=repository.get('reportSVNRevision'))
    return SVNRepository(name=repository['name'], svn_url=repository['url'], should_trust_certificate=repository.get('trustCertificate', False),
        use_server_auth=repository.get('useServerAuth', False), account_name_script_path=repository.get('accountNameFinderScript'))


class Repository(object):
    ___metaclass___ = ABCMeta

    _name_account_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<account>.+)\>\s*$')

    def __init__(self, name):
        self._name = name
        self._last_fetched = None

    def fetch_commits_and_submit(self, server_config, max_fetch_count, max_ancestor_fetch_count):
        if not self._last_fetched:
            print "Determining the starting revision for %s" % self._name
            self._last_fetched = self.determine_last_reported_revision(server_config)

        pending_commits = []
        for unused in range(max_fetch_count):
            commit = self.fetch_next_commit(server_config, self._last_fetched)
            if not commit:
                break
            pending_commits += [commit]
            self._last_fetched = commit['revision']

        if not pending_commits:
            print "No new revision found for %s (last fetched: %s)" % (self._name, self.format_revision(self._last_fetched))
            return

        for unused in range(max_ancestor_fetch_count):
            revision_list = ', '.join([self.format_revision(commit['revision']) for commit in pending_commits])
            print "Submitting revisions %s for %s to %s" % (revision_list, self._name, server_config['server']['url'])

            result = submit_commits(pending_commits, server_config['server']['url'],
                server_config['worker']['name'], server_config['worker']['password'], ['OK', 'FailedToFindPreviousCommit'])

            if result.get('status') == 'OK':
                break

            if result.get('status') == 'FailedToFindPreviousCommit':
                previous_commit = self.fetch_commit(server_config, result['commit']['previousCommit'])
                if not previous_commit:
                    raise Exception('Could not find the previous commit %s of %s' % (result['commit']['previousCommit'], result['commit']['revision']))
                pending_commits = [previous_commit] + pending_commits

        if result.get('status') != 'OK':
            raise Exception(result)

        print "Successfully submitted."
        print

    @abstractmethod
    def fetch_next_commit(self, server_config, last_fetched):
        pass

    @abstractmethod
    def fetch_commit(self, server_config, last_fetched):
        pass

    @abstractmethod
    def format_revision(self, revision):
        pass

    def determine_last_reported_revision(self, server_config):
        last_reported_revision = self.fetch_revision_from_dasbhoard(server_config, 'last-reported')
        if last_reported_revision:
            return last_reported_revision

    def fetch_revision_from_dasbhoard(self, server_config, filter):
        result = urllib2.urlopen(server_config['server']['url'] + '/api/commits/' + self._name + '/' + filter).read()
        parsed_result = json.loads(result)
        if parsed_result['status'] != 'OK' and parsed_result['status'] != 'RepositoryNotFound':
            raise Exception(result)
        commits = parsed_result.get('commits')
        return commits[0]['revision'] if commits else None


class SVNRepository(Repository):

    def __init__(self, name, svn_url, should_trust_certificate, use_server_auth, account_name_script_path):
        assert not account_name_script_path or isinstance(account_name_script_path, list)
        super(SVNRepository, self).__init__(name)
        self._svn_url = svn_url
        self._should_trust_certificate = should_trust_certificate
        self._use_server_auth = use_server_auth
        self._account_name_script_path = account_name_script_path

    def fetch_next_commit(self, server_config, last_fetched):
        if not last_fetched:
            # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
            # in the future because we never refetch older revisions.
            last_fetched = self.fetch_revision_from_dasbhoard(server_config, 'oldest')

        revision_to_fetch = int(last_fetched) + 1

        args = ['svn', 'log', '--revision', str(revision_to_fetch), '--xml', self._svn_url, '--non-interactive']
        if self._use_server_auth and 'auth' in server_config['server']:
            server_auth = server_config['server']['auth']
            args += ['--no-auth-cache', '--username', server_auth['username'], '--password', server_auth['password']]
        if self._should_trust_certificate:
            args += ['--trust-server-cert']

        try:
            output = subprocess.check_output(args, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as error:
            if (': No such revision ' + str(revision_to_fetch)) in error.output:
                return None
            raise error

        xml = parseXmlString(output)
        time = text_content(xml.getElementsByTagName("date")[0])
        author_elements = xml.getElementsByTagName("author")
        author_account = text_content(author_elements[0]) if author_elements.length else None
        message = text_content(xml.getElementsByTagName("msg")[0])

        name = self._resolve_author_name(author_account) if author_account and self._account_name_script_path else None

        result = {
            'repository': self._name,
            'revision': revision_to_fetch,
            'time': time,
            'message': message,
        }

        if author_account:
            result['author'] = {'account': author_account, 'name': name}

        return result

    def _resolve_author_name(self, account):
        try:
            output = subprocess.check_output(self._account_name_script_path + [account])
        except subprocess.CalledProcessError:
            print 'Failed to resolve the name for account:', account
            return None

        match = Repository._name_account_compound_regex.match(output)
        if match:
            return match.group('name').strip('"')
        return output.strip()

    def format_revision(self, revision):
        return 'r' + str(revision)


class GitRepository(Repository):

    def __init__(self, name, git_checkout, git_url, git_branch=None, report_revision_identifier_in_commit_msg=False, report_svn_revison=False):
        assert(os.path.isdir(git_checkout))
        super(GitRepository, self).__init__(name)
        self._git_checkout = git_checkout
        self._git_url = git_url
        self._git_branch = git_branch
        self._tokenized_hashes = []
        self._report_revision_identifier_in_commit_msg = report_revision_identifier_in_commit_msg
        self._report_svn_revision = report_svn_revison

    def fetch_next_commit(self, server_config, last_fetched):
        if not last_fetched:
            self._fetch_all_hashes()
            tokens = self._tokenized_hashes[0]
        else:
            if self._report_svn_revision:
                last_fetched_git_hash = self._git_hash_from_svn_revision(last_fetched)
                if not last_fetched_git_hash:
                    self._fetch_remote()
                    last_fetched_git_hash = self._git_hash_from_svn_revision(last_fetched)
                    if not last_fetched_git_hash:
                        raise ValueError('Cannot find the git hash for the last fetched svn revision')
                last_fetched = last_fetched_git_hash
            tokens = self._find_next_hash(last_fetched)
            if not tokens:
                self._fetch_all_hashes()
                tokens = self._find_next_hash(last_fetched)
                if not tokens:
                    return None
        return self._revision_from_tokens(tokens)

    def fetch_commit(self, server_config, hash_to_find):
        assert(self._tokenized_hashes)
        for i, tokens in enumerate(self._tokenized_hashes):
            if tokens and tokens[0] == hash_to_find:
                return self._revision_from_tokens(tokens)
        return None

    def _svn_revision_from_git_hash(self, git_hash):
        return self._run_git_command(['svn', 'find-rev', git_hash]).strip()

    def _git_hash_from_svn_revision(self, revision):
        return self._run_git_command(['svn', 'find-rev', 'r{}'.format(revision)]).strip()

    def _revision_from_tokens(self, tokens):
        current_hash = tokens[0]
        commit_time = int(tokens[1])
        author_email = tokens[2]
        previous_hash = tokens[3] if len(tokens) >= 4 else None

        author_name = self._run_git_command(['log', current_hash, '-1', '--pretty=%cn'])
        message = self._run_git_command(['log', current_hash, '-1', '--pretty=%B'])

        revision_identifier = None
        if self._report_revision_identifier_in_commit_msg:
            revision_identifier_match = REVISION_IDENTIFIER_RE.search(message)
            if not revision_identifier_match:
                raise ValueError('Expected commit message to include revision identifier, but cannot find it, will need a history rewrite to fix it')
            revision_identifier = revision_identifier_match.group('revision_identifier')

        current_revision = current_hash
        previous_revision = previous_hash
        if self._report_svn_revision:
            current_revision = self._svn_revision_from_git_hash(current_hash)
            if not current_revision:
                raise ValueError('Cannot find SVN revison for {}'.format(current_hash))
            if previous_hash:
                previous_revision = self._svn_revision_from_git_hash(previous_hash)
                if not previous_revision:
                    raise ValueError('Cannot find SVN revison for {}'.format(previous_hash))

        return {
            'repository': self._name,
            'revision': current_revision,
            'revisionIdentifier': revision_identifier,
            'previousCommit': previous_revision,
            'time': datetime.utcfromtimestamp(commit_time).strftime(r'%Y-%m-%dT%H:%M:%S.%f'),
            'author': {'account': author_email, 'name': author_name},
            'message': message,
        }

    def _find_next_hash(self, hash_to_find):
        for i, tokens in enumerate(self._tokenized_hashes):
            if tokens and tokens[0] == hash_to_find:
                return self._tokenized_hashes[i + 1] if i + 1 < len(self._tokenized_hashes) else None
        return None

    def _fetch_remote(self):
        if self._report_svn_revision:
            self._run_git_command(['pull'])
            subprocess.check_call(['rm', '-rf', os.path.join(self._git_checkout, '.git/svn')])
            self._run_git_command(['svn', 'fetch'])
        else:
            self._run_git_command(['pull', self._git_url])

    def _fetch_all_hashes(self):
        self._fetch_remote()
        scope = self._git_branch or '--all'
        lines = self._run_git_command(['log', scope, '--date-order', '--reverse', '--pretty=%H %ct %ce %P']).split('\n')
        self._tokenized_hashes = [line.split() for line in lines]

    def _run_git_command(self, args):
        return subprocess.check_output(['git', '-C', self._git_checkout] + args, stderr=subprocess.STDOUT)

    def format_revision(self, revision):
        return str(revision)[0:8]


if __name__ == "__main__":
    main(sys.argv)