# Copyright (C) 2018 Igalia S.L. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest from webkitpy.common.host_mock import MockHost from webkitpy.common.test_expectations import TestExpectations class MockTestExpectations(TestExpectations): def __init__(self, port, expectations, build_type='Release'): host = MockHost() port = host.port_factory.get(port) self._port_name = port.name() self._build_type = build_type self._expectations = self._load_expectation_string(expectations) def is_skip(self, test, subtest): if test in self.skipped_tests(): return True return subtest in self.skipped_subtests(test) class ExpectationsTest(unittest.TestCase): BASIC = """ { "imported/w3c/webdriver/tests/test1.py": { "subtests": { "test1_one": { "expected": {"all": {"status": ["FAIL"], "bug": "1234"}} }, "test1_two": { "expected": { "all": {"status": ["FAIL"], "bug": "1234"}, "gtk": {"status": ["PASS"]} } } } }, "imported/w3c/webdriver/tests/test2.py": { "expected": {"all": {"status": ["FAIL"], "bug": "1234"}} }, "imported/w3c/webdriver/tests/test3.py": { "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}, "subtests": { "test3_one": { "expected": {"all": {"status": ["PASS"]}} } } } }""" SKIP = """ { "imported/w3c/webdriver/tests/test1.py": { "expected": {"all": {"status": ["SKIP"], "bug": "1234"}} }, "imported/w3c/webdriver/tests/test2.py": { "expected": {"gtk": {"status": ["SKIP"], "bug": "1234"}} }, "imported/w3c/webdriver/tests/test3.py": { "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}, "subtests": { "test3_one": { "expected": {"all": {"status": ["FAIL"], "bug": "1234"}} } } }, "imported/w3c/webdriver/tests/test4.py": { "subtests": { "test4_one": { "expected": {"all": {"status": ["SKIP"], "bug": "1234"}} } } } }""" FLAKY = """ { "TestCookieManager": { "subtests": { "/webkit2/WebKitCookieManager/persistent-storage": { "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "1234"}} } } }, "TestWebKit": { "subtests": { "WebKit.MouseMoveAfterCrash": { "expected": {"all": {"status": ["CRASH", "PASS"], "bug": "1234"}} }, "WebKit.WKConnection": { "expected": {"wpe": {"status": ["FAIL", "TIMEOUT"], "bug": "1234"}} } } } }""" BUILD_TYPE = """ { "TestCookieManager": { "subtests": { "/webkit2/WebKitCookieManager/persistent-storage": { "expected": {"all": {"status": ["FAIL"], "bug": "1234"}} } } }, "TestWebKit": { "subtests": { "WebKit.MouseMoveAfterCrash": { "expected": {"all@Debug": {"status": ["CRASH"], "bug": "1234"}} }, "WebKit.WKConnection": { "expected": {"gtk@Release": {"status": ["FAIL"], "bug": "1234"}, "gtk@Debug": {"status": ["CRASH"], "bug": "1234"}} } } }, "TestWebCore": { "expected": {"all@Debug": {"status": ["CRASH"]}}, "subtests": { "ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL": { "expected": {"all": {"status": ["PASS"], "bug": "1234"}} }, "FileMonitorTest.DetectChange": { "expected": {"all@Release": {"status": ["FAIL"], "bug": "1234"}} } } }, "TestWebViewEditor": { "expected": {"all@Release": {"status": ["SKIP"]}, "wpe@Debug": {"status": ["SKIP"]}}, "subtests": { "/webkit2/WebKitWebView/editable/editable": { "expected": {"gtk": {"status": ["FAIL"], "bug": "1234"}} } } } }""" SLOW = """ { "TestCookieManager": { "expected": {"all": {"slow": true}}, "subtests": { "/webkit2/WebKitCookieManager/persistent-storage": { "expected": {"wpe": {"status": ["FAIL"], "slow": false, "bug": "1234"}} } } }, "TestWebKit": { "subtests": { "WebKit.MouseMoveAfterCrash": { "expected": {"all": {"status": ["FAIL"], "slow": true, "bug": "1234"}} }, "WebKit.WKConnection": { "expected": {"gtk": {"status": ["CRASH"], "bug": "1234"}} } } } }""" REPEATED_KEYS = """ { "TestDummy": { "a": 1 }, "TestAnother": { }, "TestDummy": { "a": 2 } }""" def assert_exp(self, test, subtest, result): self.assertIn(result, self.expectations.get_expectation(test, subtest)) def assert_not_exp(self, test, subtest, result): self.assertNotIn(result, self.expectations.get_expectation(test, subtest)) def assert_bad_exp(self, test): self.assertRaises(AssertionError, self.expectations.get_expectation(test)) def assert_skip(self, test, subtest, result): self.assertEqual(self.expectations.is_skip(test, subtest), result) def assert_slow(self, test, subtest, result): self.assertEqual(self.expectations.is_slow(test, subtest), result) def test_basic(self): self.expectations = MockTestExpectations('gtk', self.BASIC) self.assert_exp('imported/w3c/webdriver/tests/test5.py', 'test5_two', 'PASS') self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_five', 'PASS') self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_one', 'FAIL') self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'PASS') self.assert_exp('imported/w3c/webdriver/tests/test2.py', 'test2_one', 'FAIL') self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_two', 'FAIL') self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_one', 'PASS') self.expectations = MockTestExpectations('wpe', self.BASIC) self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'FAIL') def test_skip(self): self.expectations = MockTestExpectations('gtk', self.SKIP) self.assert_skip('imported/w3c/webdriver/tests/test5.py', None, False) self.assert_skip('imported/w3c/webdriver/tests/test1.py', None, True) self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, True) self.assert_skip('imported/w3c/webdriver/tests/test3.py', None, True) self.assert_skip('imported/w3c/webdriver/tests/test3.py', 'test3_one', True) self.assert_skip('imported/w3c/webdriver/tests/test4.py', None, False) self.assert_skip('imported/w3c/webdriver/tests/test4.py', 'test4_one', True) self.assert_exp('imported/w3c/webdriver/tests/test4.py', 'test4_one', 'SKIP') self.expectations = MockTestExpectations('wpe', self.SKIP) self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, False) def test_flaky(self): self.expectations = MockTestExpectations('gtk', self.FLAKY) self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS') self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL') self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH') self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS') self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'PASS') self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL') self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT') self.expectations = MockTestExpectations('wpe', self.FLAKY) self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS') self.assert_not_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL') self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH') self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS') self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL') self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT') def test_build_type(self): self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Debug') self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL') self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH') self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'CRASH') self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS') self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'CRASH') self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'CRASH') self.assert_skip('TestWebViewEditor', None, False) self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', False) self.assert_exp('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', 'FAIL') self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Release') self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL') self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS') self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL') self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS') self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'FAIL') self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'PASS') self.assert_skip('TestWebViewEditor', None, True) self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True) self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Release') self.assert_skip('TestWebViewEditor', None, True) self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True) self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Debug') self.assert_skip('TestWebViewEditor', None, True) self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True) def test_slow(self): self.expectations = MockTestExpectations('gtk', self.SLOW) self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/basic', True) self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', True) self.assert_slow('TestWebKit', 'WebKit.WKView', False) self.assert_slow('TestWebKit', 'WebKit.MouseMoveAfterCrash', True) self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'FAIL') self.assert_slow('TestWebKit', 'WebKit.WKConnection', False) self.expectations = MockTestExpectations('wpe', self.SLOW) self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/basic', True) self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', False) self.assert_slow('TestWebKit', 'WebKit.WKView', False) self.assert_slow('TestWebKit', 'WebKit.MouseMoveAfterCrash', True) self.assert_slow('TestWebKit', 'WebKit.WKConnection', False) def test_repeated_keys(self): self.assertRaises(ValueError, lambda: MockTestExpectations('gtk', self.REPEATED_KEYS))