81 lines
3.8 KiB
Python
Executable File
81 lines
3.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright (C) 2013 Apple Inc. All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
|
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
|
|
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
# THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Compare layout test results between two configurations of WebKit.")
|
|
|
|
parser.add_argument("--comparison", action="store", choices=["remote-layer-tree", "accelerated-drawing", "webkit2"], required=True, help="configuration to test")
|
|
parser.add_argument("-2", "--webkit2", action="store_true", default=False, help="use WebKit2 always (some configurations may force this)")
|
|
parser.add_argument('tests', metavar='TESTS', type=str, help='list of LayoutTest subdirectories to run', default=None, nargs="*")
|
|
|
|
args = parser.parse_args()
|
|
|
|
force_webkit2 = args.webkit2
|
|
if args.comparison == "remote-layer-tree":
|
|
print("Forcing use of WebKit2, as the remote layer tree depends on WebKit2.")
|
|
force_webkit2 = True
|
|
|
|
if args.comparison == "webkit2" and force_webkit2:
|
|
print("It doesn't make sense to test WebKit1 vs. WebKit2 *and* force WebKit2 on.")
|
|
sys.exit(1)
|
|
|
|
configuration_flag = flag_for_comparison(args.comparison)
|
|
|
|
results_directory = tempfile.mkdtemp(prefix="webkit-comparison-results-", suffix="-" + args.comparison)
|
|
|
|
# Run first in the default configuration, generating new baselines, then run
|
|
# again against those results in the configuration being tested.
|
|
run_webkit_tests(tests=args.tests, results_directory=results_directory, use_webkit2=force_webkit2, additional_arguments=["--no-show-results", "--new-baseline"])
|
|
run_webkit_tests(tests=args.tests, results_directory=results_directory, additional_arguments=[configuration_flag], use_webkit2=force_webkit2)
|
|
|
|
def flag_for_comparison(comparison_name):
|
|
if comparison_name == "remote-layer-tree":
|
|
return "--remote-layer-tree"
|
|
if comparison_name == "accessibility-isolated-tree":
|
|
return "--accessibility-isolated-tree"
|
|
if comparison_name == "accelerated-drawing":
|
|
return "--accelerated-drawing"
|
|
if comparison_name == "webkit2":
|
|
return "-2"
|
|
|
|
print("Unknown comparison:", comparison_name)
|
|
sys.exit(1)
|
|
|
|
def run_webkit_tests(tests, results_directory, use_webkit2, additional_arguments=[]):
|
|
if use_webkit2:
|
|
use_webkit2_arguments = ["-2"]
|
|
else:
|
|
use_webkit2_arguments = []
|
|
|
|
subprocess.call(["run-webkit-tests", "-p", "--force", "--treat-ref-tests-as-pixel-tests", "--additional-platform-directory=" + results_directory, "--run-singly", "--no-retry-failures"] + use_webkit2_arguments + additional_arguments + tests)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|