307 lines
13 KiB
Python
307 lines
13 KiB
Python
# Copyright (C) 2018 Igalia S.L.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
|
|
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
|
|
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
|
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
import unittest
|
|
|
|
from webkitpy.common.host_mock import MockHost
|
|
from webkitpy.common.test_expectations import TestExpectations
|
|
|
|
|
|
class MockTestExpectations(TestExpectations):
|
|
|
|
def __init__(self, port, expectations, build_type='Release'):
|
|
host = MockHost()
|
|
port = host.port_factory.get(port)
|
|
self._port_name = port.name()
|
|
self._build_type = build_type
|
|
self._expectations = self._load_expectation_string(expectations)
|
|
|
|
def is_skip(self, test, subtest):
|
|
if test in self.skipped_tests():
|
|
return True
|
|
return subtest in self.skipped_subtests(test)
|
|
|
|
|
|
class ExpectationsTest(unittest.TestCase):
|
|
|
|
BASIC = """
|
|
{
|
|
"imported/w3c/webdriver/tests/test1.py": {
|
|
"subtests": {
|
|
"test1_one": {
|
|
"expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
|
|
},
|
|
"test1_two": {
|
|
"expected": {
|
|
"all": {"status": ["FAIL"], "bug": "1234"},
|
|
"gtk": {"status": ["PASS"]}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"imported/w3c/webdriver/tests/test2.py": {
|
|
"expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
|
|
},
|
|
"imported/w3c/webdriver/tests/test3.py": {
|
|
"expected": {"all": {"status": ["FAIL"], "bug": "1234"}},
|
|
"subtests": {
|
|
"test3_one": {
|
|
"expected": {"all": {"status": ["PASS"]}}
|
|
}
|
|
}
|
|
}
|
|
}"""
|
|
|
|
SKIP = """
|
|
{
|
|
"imported/w3c/webdriver/tests/test1.py": {
|
|
"expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
|
|
},
|
|
"imported/w3c/webdriver/tests/test2.py": {
|
|
"expected": {"gtk": {"status": ["SKIP"], "bug": "1234"}}
|
|
},
|
|
"imported/w3c/webdriver/tests/test3.py": {
|
|
"expected": {"all": {"status": ["SKIP"], "bug": "1234"}},
|
|
"subtests": {
|
|
"test3_one": {
|
|
"expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
|
|
}
|
|
}
|
|
},
|
|
"imported/w3c/webdriver/tests/test4.py": {
|
|
"subtests": {
|
|
"test4_one": {
|
|
"expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
|
|
}
|
|
}
|
|
}
|
|
}"""
|
|
|
|
FLAKY = """
|
|
{
|
|
"TestCookieManager": {
|
|
"subtests": {
|
|
"/webkit2/WebKitCookieManager/persistent-storage": {
|
|
"expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "1234"}}
|
|
}
|
|
}
|
|
},
|
|
"TestWebKit": {
|
|
"subtests": {
|
|
"WebKit.MouseMoveAfterCrash": {
|
|
"expected": {"all": {"status": ["CRASH", "PASS"], "bug": "1234"}}
|
|
},
|
|
"WebKit.WKConnection": {
|
|
"expected": {"wpe": {"status": ["FAIL", "TIMEOUT"], "bug": "1234"}}
|
|
}
|
|
}
|
|
}
|
|
}"""
|
|
|
|
BUILD_TYPE = """
|
|
{
|
|
"TestCookieManager": {
|
|
"subtests": {
|
|
"/webkit2/WebKitCookieManager/persistent-storage": {
|
|
"expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
|
|
}
|
|
}
|
|
},
|
|
"TestWebKit": {
|
|
"subtests": {
|
|
"WebKit.MouseMoveAfterCrash": {
|
|
"expected": {"all@Debug": {"status": ["CRASH"], "bug": "1234"}}
|
|
},
|
|
"WebKit.WKConnection": {
|
|
"expected": {"gtk@Release": {"status": ["FAIL"], "bug": "1234"},
|
|
"gtk@Debug": {"status": ["CRASH"], "bug": "1234"}}
|
|
}
|
|
}
|
|
},
|
|
"TestWebCore": {
|
|
"expected": {"all@Debug": {"status": ["CRASH"]}},
|
|
"subtests": {
|
|
"ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL": {
|
|
"expected": {"all": {"status": ["PASS"], "bug": "1234"}}
|
|
},
|
|
"FileMonitorTest.DetectChange": {
|
|
"expected": {"all@Release": {"status": ["FAIL"], "bug": "1234"}}
|
|
}
|
|
}
|
|
},
|
|
"TestWebViewEditor": {
|
|
"expected": {"all@Release": {"status": ["SKIP"]},
|
|
"wpe@Debug": {"status": ["SKIP"]}},
|
|
"subtests": {
|
|
"/webkit2/WebKitWebView/editable/editable": {
|
|
"expected": {"gtk": {"status": ["FAIL"], "bug": "1234"}}
|
|
}
|
|
}
|
|
}
|
|
}"""
|
|
|
|
SLOW = """
|
|
{
|
|
"TestCookieManager": {
|
|
"expected": {"all": {"slow": true}},
|
|
"subtests": {
|
|
"/webkit2/WebKitCookieManager/persistent-storage": {
|
|
"expected": {"wpe": {"status": ["FAIL"], "slow": false, "bug": "1234"}}
|
|
}
|
|
}
|
|
},
|
|
"TestWebKit": {
|
|
"subtests": {
|
|
"WebKit.MouseMoveAfterCrash": {
|
|
"expected": {"all": {"status": ["FAIL"], "slow": true, "bug": "1234"}}
|
|
},
|
|
"WebKit.WKConnection": {
|
|
"expected": {"gtk": {"status": ["CRASH"], "bug": "1234"}}
|
|
}
|
|
}
|
|
}
|
|
}"""
|
|
|
|
REPEATED_KEYS = """
|
|
{
|
|
"TestDummy": {
|
|
"a": 1
|
|
},
|
|
"TestAnother": {
|
|
},
|
|
"TestDummy": {
|
|
"a": 2
|
|
}
|
|
}"""
|
|
|
|
def assert_exp(self, test, subtest, result):
|
|
self.assertIn(result, self.expectations.get_expectation(test, subtest))
|
|
|
|
def assert_not_exp(self, test, subtest, result):
|
|
self.assertNotIn(result, self.expectations.get_expectation(test, subtest))
|
|
|
|
def assert_bad_exp(self, test):
|
|
self.assertRaises(AssertionError, self.expectations.get_expectation(test))
|
|
|
|
def assert_skip(self, test, subtest, result):
|
|
self.assertEqual(self.expectations.is_skip(test, subtest), result)
|
|
|
|
def assert_slow(self, test, subtest, result):
|
|
self.assertEqual(self.expectations.is_slow(test, subtest), result)
|
|
|
|
def test_basic(self):
|
|
self.expectations = MockTestExpectations('gtk', self.BASIC)
|
|
self.assert_exp('imported/w3c/webdriver/tests/test5.py', 'test5_two', 'PASS')
|
|
|
|
self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_five', 'PASS')
|
|
self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_one', 'FAIL')
|
|
self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'PASS')
|
|
|
|
self.assert_exp('imported/w3c/webdriver/tests/test2.py', 'test2_one', 'FAIL')
|
|
|
|
self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_two', 'FAIL')
|
|
self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_one', 'PASS')
|
|
|
|
self.expectations = MockTestExpectations('wpe', self.BASIC)
|
|
self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'FAIL')
|
|
|
|
def test_skip(self):
|
|
self.expectations = MockTestExpectations('gtk', self.SKIP)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test5.py', None, False)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test1.py', None, True)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, True)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test3.py', None, True)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test3.py', 'test3_one', True)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test4.py', None, False)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test4.py', 'test4_one', True)
|
|
self.assert_exp('imported/w3c/webdriver/tests/test4.py', 'test4_one', 'SKIP')
|
|
|
|
self.expectations = MockTestExpectations('wpe', self.SKIP)
|
|
self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, False)
|
|
|
|
def test_flaky(self):
|
|
self.expectations = MockTestExpectations('gtk', self.FLAKY)
|
|
self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
|
|
self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
|
|
self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'PASS')
|
|
self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
|
|
self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')
|
|
|
|
self.expectations = MockTestExpectations('wpe', self.FLAKY)
|
|
self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
|
|
self.assert_not_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
|
|
self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
|
|
self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')
|
|
|
|
def test_build_type(self):
|
|
self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Debug')
|
|
self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
|
|
self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'CRASH')
|
|
self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
|
|
self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'CRASH')
|
|
self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'CRASH')
|
|
self.assert_skip('TestWebViewEditor', None, False)
|
|
self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', False)
|
|
self.assert_exp('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', 'FAIL')
|
|
|
|
self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Release')
|
|
self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
|
|
self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
|
|
self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
|
|
self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'FAIL')
|
|
self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'PASS')
|
|
self.assert_skip('TestWebViewEditor', None, True)
|
|
self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
|
|
|
|
self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Release')
|
|
self.assert_skip('TestWebViewEditor', None, True)
|
|
self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
|
|
|
|
self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Debug')
|
|
self.assert_skip('TestWebViewEditor', None, True)
|
|
self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
|
|
|
|
def test_slow(self):
|
|
self.expectations = MockTestExpectations('gtk', self.SLOW)
|
|
self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/basic', True)
|
|
self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', True)
|
|
self.assert_slow('TestWebKit', 'WebKit.WKView', False)
|
|
self.assert_slow('TestWebKit', 'WebKit.MouseMoveAfterCrash', True)
|
|
self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'FAIL')
|
|
self.assert_slow('TestWebKit', 'WebKit.WKConnection', False)
|
|
|
|
self.expectations = MockTestExpectations('wpe', self.SLOW)
|
|
self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/basic', True)
|
|
self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', False)
|
|
self.assert_slow('TestWebKit', 'WebKit.WKView', False)
|
|
self.assert_slow('TestWebKit', 'WebKit.MouseMoveAfterCrash', True)
|
|
self.assert_slow('TestWebKit', 'WebKit.WKConnection', False)
|
|
|
|
def test_repeated_keys(self):
|
|
self.assertRaises(ValueError, lambda: MockTestExpectations('gtk', self.REPEATED_KEYS))
|