# Copyright (C) 2017-2021 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from buildbot.plugins import steps, util from buildbot.process import buildstep, factory, logobserver, properties from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY from buildbot.steps import master, shell, transfer, trigger from buildbot.steps.source.svn import SVN from twisted.internet import defer import json import os import re import socket import sys import urllib if sys.version_info < (3, 5): print('ERROR: Please use Python 3. This code is not compatible with Python 2.') sys.exit(1) BUILD_WEBKIT_HOSTNAME = 'build.webkit.org' COMMITS_INFO_URL = 'https://commits.webkit.org/' CURRENT_HOSTNAME = socket.gethostname().strip() RESULTS_WEBKIT_URL = 'https://results.webkit.org' RESULTS_SERVER_API_KEY = 'RESULTS_SERVER_API_KEY' S3URL = 'https://s3-us-west-2.amazonaws.com/' WithProperties = properties.WithProperties Interpolate = properties.Interpolate class ParseByLineLogObserver(logobserver.LineConsumerLogObserver): """A pretty wrapper for LineConsumerLogObserver to avoid repeatedly setting up generator processors.""" def __init__(self, consumeLineFunc): if not callable(consumeLineFunc): raise Exception("Error: ParseByLineLogObserver requires consumeLineFunc to be callable.") self.consumeLineFunc = consumeLineFunc super(ParseByLineLogObserver, self).__init__(self.consumeLineGenerator) def consumeLineGenerator(self): """The generator LineConsumerLogObserver expects.""" try: while True: stream, line = yield self.consumeLineFunc(line) except GeneratorExit: return class TestWithFailureCount(shell.Test): failedTestsFormatString = "%d test%s failed" def countFailures(self, cmd): return 0 def commandComplete(self, cmd): shell.Test.commandComplete(self, cmd) self.failedTestCount = self.countFailures(cmd) self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s" def evaluateCommand(self, cmd): if self.failedTestCount: return FAILURE if cmd.rc != 0: return FAILURE return SUCCESS def getResultSummary(self): status = self.name if self.results != SUCCESS: if self.failedTestCount: status = self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix) else: status += ' ({})'.format(Results[self.results]) return {'step': status} class ConfigureBuild(buildstep.BuildStep): name = "configure-build" description = ["configuring build"] descriptionDone = ["configured build"] def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, device_model, *args, **kwargs): buildstep.BuildStep.__init__(self, *args, **kwargs) self.platform = platform if platform != 'jsc-only': self.platform = platform.split('-', 1)[0] self.fullPlatform = platform self.configuration = configuration self.architecture = architecture self.buildOnly = buildOnly self.additionalArguments = additionalArguments self.device_model = device_model def start(self): self.setProperty("platform", self.platform) self.setProperty("fullPlatform", self.fullPlatform) self.setProperty("configuration", self.configuration) self.setProperty("architecture", self.architecture) self.setProperty("buildOnly", self.buildOnly) self.setProperty("additionalArguments", self.additionalArguments) self.setProperty("device_model", self.device_model) self.finished(SUCCESS) return defer.succeed(None) class CheckOutSource(SVN, object): name = 'clean-and-update-working-directory' haltOnFailure = False def __init__(self, **kwargs): kwargs['repourl'] = 'https://svn.webkit.org/repository/webkit/trunk' kwargs['mode'] = 'incremental' kwargs['logEnviron'] = False super(CheckOutSource, self).__init__(**kwargs) def getResultSummary(self): if self.results == FAILURE: self.build.addStepsAfterCurrentStep([SVNCleanup()]) if self.results != SUCCESS: return {'step': 'Failed to updated working directory'} else: return {'step': 'Cleaned and updated working directory'} class SVNCleanup(shell.ShellCommand): name = 'svn-cleanup' command = ['svn', 'cleanup'] descriptionDone = ['Run svn cleanup'] def __init__(self, **kwargs): super(SVNCleanup, self).__init__(timeout=10 * 60, logEnviron=False, **kwargs) def evaluateCommand(self, cmd): self.build.buildFinished(['svn issue, retrying build'], RETRY) return super(SVNCleanup, self).evaluateCommand(cmd) class InstallWin32Dependencies(shell.Compile): description = ["installing dependencies"] descriptionDone = ["installed dependencies"] command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"] class KillOldProcesses(shell.Compile): name = "kill-old-processes" description = ["killing old processes"] descriptionDone = ["killed old processes"] command = ["python", "./Tools/CISupport/kill-old-processes", "buildbot"] class TriggerCrashLogSubmission(shell.Compile): name = "trigger-crash-log-submission" description = ["triggering crash log submission"] descriptionDone = ["triggered crash log submission"] command = ["python3", "Tools/CISupport/trigger-crash-log-submission"] class WaitForCrashCollection(shell.Compile): name = "wait-for-crash-collection" description = ["waiting for crash collection to quiesce"] descriptionDone = ["crash collection has quiesced"] command = ["python3", "Tools/CISupport/wait-for-crash-collection", "--timeout", str(5 * 60)] class CleanBuildIfScheduled(shell.Compile): name = "delete-WebKitBuild-directory" description = ["deleting WebKitBuild directory"] descriptionDone = ["deleted WebKitBuild directory"] command = ["python3", "Tools/CISupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")] def start(self): if not self.getProperty('is_clean'): self.hideStepIf = True return SKIPPED return shell.Compile.start(self) class DeleteStaleBuildFiles(shell.Compile): name = "delete-stale-build-files" description = ["deleting stale build files"] descriptionDone = ["deleted stale build files"] command = ["python3", "Tools/CISupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")] def start(self): if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed. self.hideStepIf = True return SKIPPED return shell.Compile.start(self) class InstallWinCairoDependencies(shell.ShellCommand): name = 'wincairo-requirements' description = ['updating wincairo dependencies'] descriptionDone = ['updated wincairo dependencies'] command = ['python', './Tools/Scripts/update-webkit-wincairo-libs.py'] haltOnFailure = True class InstallGtkDependencies(shell.ShellCommand): name = "jhbuild" description = ["updating gtk dependencies"] descriptionDone = ["updated gtk dependencies"] command = ["perl", "./Tools/Scripts/update-webkitgtk-libs", WithProperties("--%(configuration)s")] haltOnFailure = True class InstallWpeDependencies(shell.ShellCommand): name = "jhbuild" description = ["updating wpe dependencies"] descriptionDone = ["updated wpe dependencies"] command = ["perl", "./Tools/Scripts/update-webkitwpe-libs", WithProperties("--%(configuration)s")] haltOnFailure = True def appendCustomBuildFlags(step, platform, fullPlatform): if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe', 'playstation', 'tvos', 'watchos',): return if 'simulator' in fullPlatform: platform = platform + '-simulator' elif platform in ['ios', 'tvos', 'watchos']: platform = platform + '-device' step.setCommand(step.command + ['--' + platform]) def appendCustomTestingFlags(step, platform, device_model): if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'): return if device_model == 'iphone': device_model = 'iphone-simulator' elif device_model == 'ipad': device_model = 'ipad-simulator' else: device_model = platform step.setCommand(step.command + ['--' + device_model]) class CompileWebKit(shell.Compile): command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")] env = {'MFLAGS': ''} name = "compile-webkit" description = ["compiling"] descriptionDone = ["compiled"] warningPattern = ".*arning: .*" def start(self): platform = self.getProperty('platform') buildOnly = self.getProperty('buildOnly') architecture = self.getProperty('architecture') additionalArguments = self.getProperty('additionalArguments') self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) if additionalArguments: self.setCommand(self.command + additionalArguments) if platform in ('mac', 'ios', 'tvos', 'watchos') and architecture: self.setCommand(self.command + ['ARCHS=' + architecture]) self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO']) if platform in ('mac', 'ios', 'tvos', 'watchos') and buildOnly: # For build-only bots, the expectation is that tests will be run on separate machines, # so we need to package debug info as dSYMs. Only generating line tables makes # this much faster than full debug info, and crash logs still have line numbers. self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']) self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=line-tables-only']) if platform == 'gtk': prefix = os.path.join("/app", "webkit", "WebKitBuild", self.getProperty("configuration").title(), "install") self.setCommand(self.command + [f'--prefix={prefix}']) appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform')) return shell.Compile.start(self) def parseOutputLine(self, line): if "arning:" in line: self._addToLog('warnings', line + '\n') if "rror:" in line: self._addToLog('errors', line + '\n') @defer.inlineCallbacks def _addToLog(self, logName, message): try: log = self.getLog(logName) except KeyError: log = yield self.addLog(logName) log.addStdout(message) class CompileLLINTCLoop(CompileWebKit): command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")] class Compile32bitJSC(CompileWebKit): command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")] class CompileJSCOnly(CompileWebKit): command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")] class InstallBuiltProduct(shell.ShellCommand): name = 'install-built-product' description = ['Installing Built Product'] descriptionDone = ['Installed Built Product'] command = ["python3", "Tools/Scripts/install-built-product", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")] class ArchiveBuiltProduct(shell.ShellCommand): command = ["python3", "Tools/CISupport/built-product-archive", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"] name = "archive-built-product" description = ["archiving built product"] descriptionDone = ["archived built product"] haltOnFailure = True class ArchiveMinifiedBuiltProduct(ArchiveBuiltProduct): command = ["python3", "Tools/CISupport/built-product-archive", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive", "--minify"] class GenerateJSCBundle(shell.ShellCommand): command = ["./Tools/Scripts/generate-bundle", "--builder-name", WithProperties("%(buildername)s"), "--bundle=jsc", "--syslibs=bundle-all", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), WithProperties("--revision=%(got_revision)s"), "--remote-config-file", "../../remote-jsc-bundle-upload-config.json"] name = "generate-jsc-bundle" description = ["generating jsc bundle"] descriptionDone = ["generated jsc bundle"] haltOnFailure = False class GenerateMiniBrowserBundle(shell.ShellCommand): command = ["./Tools/Scripts/generate-bundle", "--builder-name", WithProperties("%(buildername)s"), "--bundle=MiniBrowser", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), WithProperties("--revision=%(got_revision)s"), "--remote-config-file", "../../remote-minibrowser-bundle-upload-config.json"] name = "generate-minibrowser-bundle" description = ["generating minibrowser bundle"] descriptionDone = ["generated minibrowser bundle"] haltOnFailure = False class ExtractBuiltProduct(shell.ShellCommand): command = ["python3", "Tools/CISupport/built-product-archive", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"] name = "extract-built-product" description = ["extracting built product"] descriptionDone = ["extracted built product"] haltOnFailure = True class UploadBuiltProduct(transfer.FileUpload): workersrc = WithProperties("WebKitBuild/%(configuration)s.zip") masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") haltOnFailure = True def __init__(self, **kwargs): kwargs['workersrc'] = self.workersrc kwargs['masterdest'] = self.masterdest kwargs['mode'] = 0o644 kwargs['blocksize'] = 1024 * 256 transfer.FileUpload.__init__(self, **kwargs) class UploadMinifiedBuiltProduct(UploadBuiltProduct): workersrc = WithProperties("WebKitBuild/minified-%(configuration)s.zip") masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip") class DownloadBuiltProduct(shell.ShellCommand): command = ["python", "./Tools/CISupport/download-built-product", WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), WithProperties(S3URL + "archives.webkit.org/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")] name = "download-built-product" description = ["downloading built product"] descriptionDone = ["downloaded built product"] haltOnFailure = False flunkOnFailure = False def start(self): return shell.ShellCommand.start(self) def evaluateCommand(self, cmd): rc = shell.ShellCommand.evaluateCommand(self, cmd) if rc == FAILURE: self.build.addStepsAfterCurrentStep([DownloadBuiltProductFromMaster()]) return rc class DownloadBuiltProductFromMaster(transfer.FileDownload): mastersrc = WithProperties('archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip') workerdest = WithProperties('WebKitBuild/%(configuration)s.zip') name = 'download-built-product-from-master' description = ['downloading built product from buildbot master'] descriptionDone = ['Downloaded built product'] haltOnFailure = True flunkOnFailure = True def __init__(self, **kwargs): # Allow the unit test to override mastersrc if 'mastersrc' not in kwargs: kwargs['mastersrc'] = self.mastersrc kwargs['workerdest'] = self.workerdest kwargs['mode'] = 0o0644 kwargs['blocksize'] = 1024 * 256 transfer.FileDownload.__init__(self, **kwargs) def getResultSummary(self): if self.results != SUCCESS: return {u'step': u'Failed to download built product from build master'} return super(DownloadBuiltProductFromMaster, self).getResultSummary() class RunJavaScriptCoreTests(TestWithFailureCount): name = "jscore-test" description = ["jscore-tests running"] descriptionDone = ["jscore-tests"] jsonFileName = "jsc_results.json" command = [ "perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s"), "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--buildbot-worker", WithProperties("%(workername)s"), "--buildbot-master", CURRENT_HOSTNAME, "--report", RESULTS_WEBKIT_URL, ] failedTestsFormatString = "%d JSC test%s failed" logfiles = {"json": jsonFileName} def __init__(self, *args, **kwargs): kwargs['logEnviron'] = False TestWithFailureCount.__init__(self, *args, **kwargs) def start(self): self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) self.log_observer = logobserver.BufferLogObserver() self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 platform = self.getProperty('platform') architecture = self.getProperty("architecture") # Currently run-javascriptcore-test doesn't support run javascript core test binaries list below remotely if architecture in ['mips', 'armv7', 'aarch64']: self.command += ['--no-testmasm', '--no-testair', '--no-testb3', '--no-testdfg', '--no-testapi'] # Linux bots have currently problems with JSC tests that try to use large amounts of memory. # Check: https://bugs.webkit.org/show_bug.cgi?id=175140 if platform in ('gtk', 'wpe', 'jsc-only'): self.setCommand(self.command + ['--memory-limited', '--verbose']) # WinCairo uses the Windows command prompt, not Cygwin. elif platform == 'wincairo': self.setCommand(self.command + ['--test-writer=ruby']) appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform')) return shell.Test.start(self) def countFailures(self, cmd): logText = self.log_observer.getStdout() count = 0 match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE) if match: count += int(match.group(1)) match = re.search(r'Results for JSC test binaries:\r?\n\s+(\d+) failure', logText, re.MULTILINE) if match: count += int(match.group(1)) match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE) if match: count += int(match.group(1)) return count class RunTest262Tests(TestWithFailureCount): name = "test262-test" description = ["test262-tests running"] descriptionDone = ["test262-tests"] failedTestsFormatString = "%d Test262 test%s failed" command = ["perl", "./Tools/Scripts/test262-runner", "--verbose", WithProperties("--%(configuration)s")] test_summary_re = re.compile(r'^\! NEW FAIL') def start(self): self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform')) return shell.Test.start(self) def parseOutputLine(self, line): match = self.test_summary_re.match(line) if match: self.failedTestCount += 1 def countFailures(self, cmd): return self.failedTestCount class RunWebKitTests(shell.Test): name = "layout-test" description = ["layout-tests running"] descriptionDone = ["layout-tests"] resultDirectory = "layout-test-results" command = ["python", "./Tools/Scripts/run-webkit-tests", "--no-build", "--no-show-results", "--no-new-test-results", "--clobber-old-results", "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--buildbot-worker", WithProperties("%(workername)s"), "--buildbot-master", CURRENT_HOSTNAME, "--report", RESULTS_WEBKIT_URL, "--exit-after-n-crashes-or-timeouts", "50", "--exit-after-n-failures", "500", WithProperties("--%(configuration)s")] # FIXME: This will break if run-webkit-tests changes its default log formatter. nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P.*)') expressions = [ ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')), ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')), ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')), ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')), ] def __init__(self, *args, **kwargs): kwargs['logEnviron'] = False shell.Test.__init__(self, *args, **kwargs) def start(self): self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.incorrectLayoutLines = [] self.testFailures = {} platform = self.getProperty('platform') appendCustomTestingFlags(self, platform, self.getProperty('device_model')) additionalArguments = self.getProperty('additionalArguments') self.setCommand(self.command + ["--results-directory", self.resultDirectory]) self.setCommand(self.command + ['--debug-rwt-logging']) if platform == "win": self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin64")]) if additionalArguments: self.setCommand(self.command + additionalArguments) return shell.Test.start(self) def _strip_python_logging_prefix(self, line): match_object = self.nrwt_log_message_regexp.match(line) if match_object: return match_object.group('message') return line def parseOutputLine(self, line): if r'Exiting early' in line or r'leaks found' in line: self.incorrectLayoutLines.append(self._strip_python_logging_prefix(line)) return for name, expression in self.expressions: match = expression.search(line) if match: self.testFailures[name] = self.testFailures.get(name, 0) + int(match.group(1)) def processTestFailures(self): for name, result in self.testFailures.items(): self.incorrectLayoutLines.append(str(result) + ' ' + name) def evaluateCommand(self, cmd): self.processTestFailures() result = SUCCESS if self.incorrectLayoutLines: if len(self.incorrectLayoutLines) == 1: line = self.incorrectLayoutLines[0] if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0: return WARNINGS for line in self.incorrectLayoutLines: if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0: result = WARNINGS else: return FAILURE # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py. # This means that an exception was raised when running run-webkit-tests and # was never handled. if cmd.rc == 254: return EXCEPTION if cmd.rc != 0: return FAILURE return result def getResultSummary(self): status = self.name if self.results != SUCCESS and self.incorrectLayoutLines: status = ' '.join(self.incorrectLayoutLines) return {'step': status} return super(RunWebKitTests, self).getResultSummary() class RunDashboardTests(RunWebKitTests): name = "dashboard-tests" description = ["dashboard-tests running"] descriptionDone = ["dashboard-tests"] resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results") def start(self): self.setCommand(self.command + ["--layout-tests-directory", "./Tools/CISupport/build-webkit-org/public_html/dashboard/Scripts/tests"]) return RunWebKitTests.start(self) class RunAPITests(TestWithFailureCount): name = "run-api-tests" description = ["api tests running"] descriptionDone = ["api-tests"] jsonFileName = "api_test_results.json" logfiles = {"json": jsonFileName} command = [ "python3", "./Tools/Scripts/run-api-tests", "--no-build", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s"), "--verbose", "--buildbot-master", CURRENT_HOSTNAME, "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--buildbot-worker", WithProperties("%(workername)s"), "--report", RESULTS_WEBKIT_URL, ] failedTestsFormatString = "%d api test%s failed or timed out" test_summary_re = re.compile(r'Ran (?P\d+) tests of (?P\d+) with (?P\d+) successful') def __init__(self, *args, **kwargs): kwargs['logEnviron'] = False TestWithFailureCount.__init__(self, *args, **kwargs) def start(self): self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 appendCustomTestingFlags(self, self.getProperty('platform'), self.getProperty('device_model')) return shell.Test.start(self) def countFailures(self, cmd): return self.failedTestCount def parseOutputLine(self, line): match = self.test_summary_re.match(line) if match: self.failedTestCount = int(match.group('ran')) - int(match.group('passed')) class RunPythonTests(TestWithFailureCount): test_summary_re = re.compile(r'^FAILED \((?P[^)]+)\)') # e.g.: FAILED (failures=2, errors=1) def start(self): self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 platform = self.getProperty('platform') # Python tests are flaky on the GTK builders, running them serially # helps and does not significantly prolong the cycle time. if platform == 'gtk': self.setCommand(self.command + ['--child-processes', '1']) # Python tests fail on windows bots when running more than one child process # https://bugs.webkit.org/show_bug.cgi?id=97465 if platform == 'win': self.setCommand(self.command + ['--child-processes', '1']) return shell.Test.start(self) def parseOutputLine(self, line): match = self.test_summary_re.match(line) if match: self.failedTestCount = sum(int(component.split('=')[1]) for component in match.group('counts').split(', ')) def countFailures(self, cmd): return self.failedTestCount class RunWebKitPyTests(RunPythonTests): name = "webkitpy-test" description = ["python-tests running"] descriptionDone = ["python-tests"] command = [ "python3", "./Tools/Scripts/test-webkitpy", "--verbose", "--buildbot-master", CURRENT_HOSTNAME, "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--buildbot-worker", WithProperties("%(workername)s"), "--report", RESULTS_WEBKIT_URL, ] failedTestsFormatString = "%d python test%s failed" def __init__(self, *args, **kwargs): kwargs['logEnviron'] = False RunPythonTests.__init__(self, *args, **kwargs) def start(self): self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) return RunPythonTests.start(self) class RunLLDBWebKitTests(RunPythonTests): name = "lldb-webkit-test" description = ["lldb-webkit-tests running"] descriptionDone = ["lldb-webkit-tests"] command = [ "python3", "./Tools/Scripts/test-lldb-webkit", "--verbose", "--no-build", WithProperties("--%(configuration)s"), ] failedTestsFormatString = "%d lldb test%s failed" class RunPerlTests(TestWithFailureCount): name = "webkitperl-test" description = ["perl-tests running"] descriptionDone = ["perl-tests"] command = ["perl", "./Tools/Scripts/test-webkitperl"] failedTestsFormatString = "%d perl test%s failed" test_summary_re = re.compile(r'^Failed \d+/\d+ test programs\. (?P\d+)/\d+ subtests failed\.') # e.g.: Failed 2/19 test programs. 5/363 subtests failed. def start(self): self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 return shell.Test.start(self) def parseOutputLine(self, line): match = self.test_summary_re.match(line) if match: self.failedTestCount = int(match.group('count')) def countFailures(self, cmd): return self.failedTestCount class RunLLINTCLoopTests(TestWithFailureCount): name = "webkit-jsc-cloop-test" description = ["cloop-tests running"] descriptionDone = ["cloop-tests"] jsonFileName = "jsc_cloop.json" command = [ "perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s"), "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--buildbot-worker", WithProperties("%(workername)s"), "--buildbot-master", CURRENT_HOSTNAME, "--report", RESULTS_WEBKIT_URL, ] failedTestsFormatString = "%d regression%s found." logfiles = {"json": jsonFileName} test_summary_re = re.compile(r'\s*(?P\d+) regressions? found.') # e.g.: 2 regressions found. def __init__(self, *args, **kwargs): kwargs['logEnviron'] = False TestWithFailureCount.__init__(self, *args, **kwargs) def start(self): self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 return shell.Test.start(self) def parseOutputLine(self, line): match = self.test_summary_re.match(line) if match: self.failedTestCount = int(match.group('count')) def countFailures(self, cmd): return self.failedTestCount class Run32bitJSCTests(TestWithFailureCount): name = "webkit-32bit-jsc-test" description = ["32bit-jsc-tests running"] descriptionDone = ["32bit-jsc-tests"] jsonFileName = "jsc_32bit.json" command = [ "perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", "--no-fail-fast", "--no-jit", "--no-testair", "--no-testb3", "--no-testmasm", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s"), "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--buildbot-worker", WithProperties("%(workername)s"), "--buildbot-master", CURRENT_HOSTNAME, "--report", RESULTS_WEBKIT_URL, ] failedTestsFormatString = "%d regression%s found." logfiles = {"json": jsonFileName} test_summary_re = re.compile(r'\s*(?P\d+) failures? found.') # e.g.: 2 failures found. def __init__(self, *args, **kwargs): kwargs['logEnviron'] = False TestWithFailureCount.__init__(self, *args, **kwargs) def start(self): self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) self.log_observer = ParseByLineLogObserver(self.parseOutputLine) self.addLogObserver('stdio', self.log_observer) self.failedTestCount = 0 return shell.Test.start(self) def parseOutputLine(self, line): match = self.test_summary_re.match(line) if match: self.failedTestCount = int(match.group('count')) def countFailures(self, cmd): return self.failedTestCount class RunBindingsTests(shell.Test): name = "bindings-generation-tests" description = ["bindings-tests running"] descriptionDone = ["bindings-tests"] command = ["python", "./Tools/Scripts/run-bindings-tests"] class RunBuiltinsTests(shell.Test): name = "builtins-generator-tests" description = ["builtins-generator-tests running"] descriptionDone = ["builtins-generator-tests"] command = ["python", "./Tools/Scripts/run-builtins-generator-tests"] class RunGLibAPITests(shell.Test): name = "API-tests" description = ["API tests running"] descriptionDone = ["API tests"] def start(self): additionalArguments = self.getProperty("additionalArguments") if additionalArguments: self.command += additionalArguments self.setCommand(self.command) return shell.Test.start(self) def commandComplete(self, cmd): shell.Test.commandComplete(self, cmd) logText = cmd.logs['stdio'].getText() failedTests = 0 crashedTests = 0 timedOutTests = 0 messages = [] self.statusLine = [] foundItems = re.findall(r"Unexpected failures \((\d+)\)", logText) if foundItems: failedTests = int(foundItems[0]) messages.append("%d failures" % failedTests) foundItems = re.findall(r"Unexpected crashes \((\d+)\)", logText) if foundItems: crashedTests = int(foundItems[0]) messages.append("%d crashes" % crashedTests) foundItems = re.findall(r"Unexpected timeouts \((\d+)\)", logText) if foundItems: timedOutTests = int(foundItems[0]) messages.append("%d timeouts" % timedOutTests) foundItems = re.findall(r"Unexpected passes \((\d+)\)", logText) if foundItems: newPassTests = int(foundItems[0]) messages.append("%d new passes" % newPassTests) self.totalFailedTests = failedTests + crashedTests + timedOutTests if messages: self.statusLine = ["API tests: %s" % ", ".join(messages)] def evaluateCommand(self, cmd): if self.totalFailedTests > 0: return FAILURE if cmd.rc != 0: return FAILURE return SUCCESS def getText(self, cmd, results): return self.getText2(cmd, results) def getText2(self, cmd, results): if results != SUCCESS and self.totalFailedTests > 0: return self.statusLine return [self.name] class RunGtkAPITests(RunGLibAPITests): command = ["python", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")] class RunWPEAPITests(RunGLibAPITests): command = ["python", "./Tools/Scripts/run-wpe-tests", WithProperties("--%(configuration)s")] class RunWebDriverTests(shell.Test): name = "webdriver-test" description = ["webdriver-tests running"] descriptionDone = ["webdriver-tests"] jsonFileName = "webdriver_tests.json" command = ["python", "./Tools/Scripts/run-webdriver-tests", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")] logfiles = {"json": jsonFileName} def start(self): additionalArguments = self.getProperty('additionalArguments') if additionalArguments: self.setCommand(self.command + additionalArguments) appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform')) return shell.Test.start(self) def commandComplete(self, cmd): shell.Test.commandComplete(self, cmd) logText = cmd.logs['stdio'].getText() self.failuresCount = 0 self.newPassesCount = 0 foundItems = re.findall(r"^Unexpected .+ \((\d+)\)", logText, re.MULTILINE) if foundItems: self.failuresCount = int(foundItems[0]) foundItems = re.findall(r"^Expected to .+, but passed \((\d+)\)", logText, re.MULTILINE) if foundItems: self.newPassesCount = int(foundItems[0]) def evaluateCommand(self, cmd): if self.failuresCount: return FAILURE if self.newPassesCount: return WARNINGS if cmd.rc != 0: return FAILURE return SUCCESS def getText(self, cmd, results): return self.getText2(cmd, results) def getText2(self, cmd, results): if results != SUCCESS and (self.failuresCount or self.newPassesCount): lines = [] if self.failuresCount: lines.append("%d failures" % self.failuresCount) if self.newPassesCount: lines.append("%d new passes" % self.newPassesCount) return ["%s %s" % (self.name, ", ".join(lines))] return [self.name] class RunWebKit1Tests(RunWebKitTests): def start(self): self.setCommand(self.command + ["--dump-render-tree"]) return RunWebKitTests.start(self) class RunWebKit1LeakTests(RunWebKit1Tests): want_stdout = False want_stderr = False warnOnWarnings = True def start(self): self.setCommand(self.command + ["--leaks", "--result-report-flavor", "Leaks"]) return RunWebKit1Tests.start(self) class RunAndUploadPerfTests(shell.Test): name = "perf-test" description = ["perf-tests running"] descriptionDone = ["perf-tests"] command = ["python", "./Tools/Scripts/run-perf-tests", "--output-json-path", "perf-test-results.json", "--worker-config-json-path", "../../perf-test-config.json", "--no-show-results", "--reset-results", "--test-results-server", "perf.webkit.org", "--builder-name", WithProperties("%(buildername)s"), "--build-number", WithProperties("%(buildnumber)s"), "--platform", WithProperties("%(fullPlatform)s"), "--no-build", WithProperties("--%(configuration)s")] def start(self): additionalArguments = self.getProperty("additionalArguments") if additionalArguments: self.command += additionalArguments self.setCommand(self.command) return shell.Test.start(self) def getText(self, cmd, results): return self.getText2(cmd, results) def getText2(self, cmd, results): if results != SUCCESS: if cmd.rc == -1 & 0xff: return ["build not up to date"] elif cmd.rc == -2 & 0xff: return ["worker config JSON error"] elif cmd.rc == -3 & 0xff: return ["output JSON merge error"] elif cmd.rc == -4 & 0xff: return ["upload error"] elif cmd.rc == -5 & 0xff: return ["system dependency error"] elif cmd.rc == -1: return ["timeout"] else: return ["%d perf tests failed" % cmd.rc] return [self.name] class RunBenchmarkTests(shell.Test): name = "benchmark-test" description = ["benchmark tests running"] descriptionDone = ["benchmark tests"] command = ["python", "./Tools/Scripts/browserperfdash-benchmark", "--allplans", "--config-file", "../../browserperfdash-benchmark-config.txt", "--browser-version", WithProperties("r%(got_revision)s")] def start(self): platform = self.getProperty("platform") if platform == "gtk": self.command += ["--browser", "minibrowser-gtk"] self.setCommand(self.command) return shell.Test.start(self) def getText(self, cmd, results): return self.getText2(cmd, results) def getText2(self, cmd, results): if results != SUCCESS: return ["%d benchmark tests failed" % cmd.rc] return [self.name] class ArchiveTestResults(shell.ShellCommand): command = ["python", "./Tools/CISupport/test-result-archive", WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] name = "archive-test-results" description = ["archiving test results"] descriptionDone = ["archived test results"] haltOnFailure = True class UploadTestResults(transfer.FileUpload): workersrc = "layout-test-results.zip" masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") def __init__(self, **kwargs): kwargs['workersrc'] = self.workersrc kwargs['masterdest'] = self.masterdest kwargs['mode'] = 0o644 kwargs['blocksize'] = 1024 * 256 transfer.FileUpload.__init__(self, **kwargs) class TransferToS3(master.MasterShellCommand): name = "transfer-to-s3" description = ["transferring to s3"] descriptionDone = ["transferred to s3"] archive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") minifiedArchive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip") identifier = WithProperties("%(fullPlatform)s-%(architecture)s-%(configuration)s") revision = WithProperties("%(got_revision)s") command = ["python3", "../Shared/transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive] haltOnFailure = True def __init__(self, **kwargs): kwargs['command'] = self.command kwargs['logEnviron'] = False master.MasterShellCommand.__init__(self, **kwargs) def start(self): return master.MasterShellCommand.start(self) def finished(self, result): return master.MasterShellCommand.finished(self, result) def doStepIf(self, step): return CURRENT_HOSTNAME == BUILD_WEBKIT_HOSTNAME class ExtractTestResults(master.MasterShellCommand): name = 'extract-test-results' descriptionDone = ['Extracted test results'] renderables = ['resultDirectory', 'zipFile'] def __init__(self, **kwargs): kwargs['command'] = "" kwargs['logEnviron'] = False self.zipFile = Interpolate('public_html/results/%(prop:buildername)s/r%(prop:got_revision)s (%(prop:buildnumber)s).zip') self.resultDirectory = Interpolate('public_html/results/%(prop:buildername)s/r%(prop:got_revision)s (%(prop:buildnumber)s)') kwargs['command'] = ['unzip', '-q', '-o', self.zipFile, '-d', self.resultDirectory] master.MasterShellCommand.__init__(self, **kwargs) def resultDirectoryURL(self): self.setProperty('result_directory', self.resultDirectory) return self.resultDirectory.replace('public_html/', '/') + '/' def addCustomURLs(self): self.addURL("view layout test results", self.resultDirectoryURL() + "results.html") self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html") def finished(self, result): self.addCustomURLs() return master.MasterShellCommand.finished(self, result) class PrintConfiguration(steps.ShellSequence): name = 'configuration' description = ['configuration'] haltOnFailure = False flunkOnFailure = False warnOnFailure = False logEnviron = False command_list_generic = [['hostname']] command_list_apple = [['df', '-hl'], ['date'], ['sw_vers'], ['xcodebuild', '-sdk', '-version'], ['uptime']] command_list_linux = [['df', '-hl'], ['date'], ['uname', '-a'], ['uptime']] command_list_win = [['df', '-hl']] def __init__(self, **kwargs): super(PrintConfiguration, self).__init__(timeout=60, **kwargs) self.commands = [] self.log_observer = logobserver.BufferLogObserver(wantStderr=True) self.addLogObserver('stdio', self.log_observer) def run(self): command_list = list(self.command_list_generic) platform = self.getProperty('platform', '*') if platform != 'jsc-only': platform = platform.split('-')[0] if platform in ('mac', 'ios', 'tvos', 'watchos', '*'): command_list.extend(self.command_list_apple) elif platform in ('gtk', 'wpe', 'jsc-only'): command_list.extend(self.command_list_linux) elif platform in ('win'): command_list.extend(self.command_list_win) for command in command_list: self.commands.append(util.ShellArg(command=command, logname='stdio')) return super(PrintConfiguration, self).run() def convert_build_to_os_name(self, build): if not build: return 'Unknown' build_to_name_mapping = { '11': 'Big Sur', '10.15': 'Catalina', '10.14': 'Mojave', '10.13': 'High Sierra', '10.12': 'Sierra', '10.11': 'El Capitan', '10.10': 'Yosemite', '10.9': 'Maverick', '10.8': 'Mountain Lion', '10.7': 'Lion', '10.6': 'Snow Leopard', '10.5': 'Leopard', } for key, value in build_to_name_mapping.items(): if build.startswith(key): return value return 'Unknown' def getResultSummary(self): if self.results != SUCCESS: return {'step': 'Failed to print configuration'} logText = self.log_observer.getStdout() + self.log_observer.getStderr() configuration = 'Printed configuration' match = re.search('ProductVersion:[ \t]*(.+?)\n', logText) if match: os_version = match.group(1).strip() os_name = self.convert_build_to_os_name(os_version) configuration = 'OS: {} ({})'.format(os_name, os_version) xcode_re = sdk_re = 'Xcode[ \t]+?([0-9.]+?)\n' match = re.search(xcode_re, logText) if match: xcode_version = match.group(1).strip() configuration += ', Xcode: {}'.format(xcode_version) return {'step': configuration} class SetPermissions(master.MasterShellCommand): name = 'set-permissions' def __init__(self, **kwargs): resultDirectory = Interpolate('%(prop:result_directory)s') kwargs['command'] = ['chmod', 'a+rx', resultDirectory] kwargs['logEnviron'] = False master.MasterShellCommand.__init__(self, **kwargs) class ShowIdentifier(shell.ShellCommand): name = 'show-identifier' identifier_re = '^Identifier: (.*)$' flunkOnFailure = False haltOnFailure = False def __init__(self, **kwargs): shell.ShellCommand.__init__(self, timeout=10 * 60, logEnviron=False, **kwargs) def start(self): self.log_observer = logobserver.BufferLogObserver() self.addLogObserver('stdio', self.log_observer) revision = self.getProperty('got_revision') self.setCommand(['python', 'Tools/Scripts/git-webkit', 'find', 'r{}'.format(revision)]) return shell.ShellCommand.start(self) def evaluateCommand(self, cmd): rc = shell.ShellCommand.evaluateCommand(self, cmd) if rc != SUCCESS: return rc log_text = self.log_observer.getStdout() match = re.search(self.identifier_re, log_text, re.MULTILINE) if match: identifier = match.group(1) if identifier: identifier = identifier.replace('trunk', 'main') self.setProperty('identifier', identifier) step = self.getLastBuildStepByName(CheckOutSource.name) if not step: step = self step.addURL('Updated to {}'.format(identifier), self.url_for_identifier(identifier)) self.descriptionDone = 'Identifier: {}'.format(identifier) else: self.descriptionDone = 'Failed to find identifier' return rc def getLastBuildStepByName(self, name): for step in reversed(self.build.executedSteps): if name in step.name: return step return None def url_for_identifier(self, identifier): return '{}{}'.format(COMMITS_INFO_URL, identifier) def getResultSummary(self): if self.results != SUCCESS: return {u'step': u'Failed to find identifier'} return shell.ShellCommand.getResultSummary(self) def hideStepIf(self, results, step): return results == SUCCESS