404 lines
20 KiB
Python
Executable File
404 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2017, 2021 Igalia S.L.
|
|
# Copyright (C) 2020 Apple Inc. All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
#
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the following disclaimer
|
|
# in the documentation and/or other materials provided with the
|
|
# distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
import sys
|
|
import signal
|
|
import os
|
|
import argparse
|
|
import subprocess
|
|
import tempfile
|
|
import shutil
|
|
import socket
|
|
import json
|
|
import traceback
|
|
import multiprocessing
|
|
from time import sleep
|
|
|
|
buildbot_server_tac = """
|
|
import os
|
|
from twisted.application import service
|
|
from buildbot.master import BuildMaster
|
|
|
|
basedir = os.path.dirname(os.path.realpath(__file__))
|
|
configfile = r'%(config_file)s'
|
|
|
|
application = service.Application('buildmaster')
|
|
BuildMaster(basedir, configfile).setServiceParent(application)
|
|
"""
|
|
|
|
buildbot_worker_tac = """
|
|
import os
|
|
from buildbot_worker.bot import Worker
|
|
from twisted.application import service
|
|
|
|
basedir = os.path.dirname(os.path.realpath(__file__))
|
|
application = service.Application('buildbot-worker')
|
|
|
|
buildmaster_host = 'localhost'
|
|
port = %(server_pb_port)s
|
|
workername = '%(worker_name)s'
|
|
passwd = 'password'
|
|
keepalive = 600
|
|
|
|
s = Worker(buildmaster_host, port, workername, passwd, basedir, keepalive)
|
|
s.setServiceParent(application)
|
|
"""
|
|
|
|
|
|
def check_tcp_port_open(address, port):
|
|
s = socket.socket()
|
|
try:
|
|
s.connect((address, port))
|
|
return True
|
|
except ConnectionRefusedError:
|
|
return False
|
|
|
|
|
|
def create_tempdir(tmpdir=None):
|
|
if tmpdir is not None:
|
|
if not os.path.isdir(tmpdir):
|
|
raise ValueError('{} is not a directory'.format(tmpdir))
|
|
return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(tmpdir), 'tmp'))
|
|
return tempfile.mkdtemp()
|
|
|
|
|
|
def print_if_error_stdout_stderr(cmd, retcode, stdout=None, stderr=None, extramsg=None):
|
|
if retcode != 0:
|
|
if type(cmd) is list:
|
|
cmd = ' '.join(cmd)
|
|
print('WARNING: "{cmd}" returned {retcode} status code'.format(cmd=cmd, retcode=retcode))
|
|
if stdout is not None:
|
|
print('STDOUT:\n' + stdout.decode('utf-8'))
|
|
if stderr is not None:
|
|
print('STDERR:\n' + stderr.decode('utf-8'))
|
|
if extramsg is not None:
|
|
print(extramsg)
|
|
|
|
|
|
def cmd_exists(cmd):
|
|
return any(os.access(os.path.join(path, cmd), os.X_OK)
|
|
for path in os.environ['PATH'].split(os.pathsep))
|
|
|
|
|
|
class BuildbotTestRunner(object):
|
|
|
|
def __init__(self, configdir):
|
|
self._configdir = os.path.abspath(os.path.realpath(configdir))
|
|
if not os.path.isdir(self._configdir):
|
|
raise RuntimeError('The configdir {} is not a directory'.format(self._configdir))
|
|
if not os.path.isfile(os.path.join(self._configdir, 'config.json')):
|
|
raise RuntimeError('The configdir {} does not contain a config.json file'.format(self._configdir))
|
|
|
|
number_config_files = 0
|
|
for file in os.listdir(self._configdir):
|
|
if file.endswith('.cfg'):
|
|
self._server_config_file_name = file
|
|
number_config_files += 1
|
|
if number_config_files == 0:
|
|
raise RuntimeError('The configdir {} does not contain a .cfg file'.format(self._configdir))
|
|
if number_config_files != 1:
|
|
raise RuntimeError('The configdir {} has more than one .cfg file'.format(self._configdir))
|
|
|
|
self._server_http_port, self._server_pb_port = self._get_config_tcp_ports()
|
|
|
|
def _get_config_tcp_ports(self):
|
|
pb_port = 17000
|
|
http_port = 8010
|
|
try:
|
|
with open(os.path.join(self._configdir, self._server_config_file_name)) as f:
|
|
for line in f:
|
|
if '=' in line:
|
|
if 'protocols' in line and 'pb' in line and 'port' in line:
|
|
pb = eval(line.split('=', 1)[1])
|
|
pb_port = int((pb['pb']['port']))
|
|
if 'www' in line and 'port' in line:
|
|
http = eval(line.split('=', 1)[1])
|
|
http_port = int((http['port']))
|
|
except:
|
|
print("Unable to detect http and pb ports from config. Using defaults")
|
|
return http_port, pb_port
|
|
|
|
def start(self, basetempdir=None, no_clean=False, number_workers=1, use_system_version=False):
|
|
try:
|
|
self._base_workdir_temp = os.path.abspath(os.path.realpath(create_tempdir(basetempdir)))
|
|
if self._base_workdir_temp.startswith(self._configdir):
|
|
raise ValueError('The temporal working directory {} cant be located inside configdir {}'.format(self._base_workdir_temp, self._configdir))
|
|
if not use_system_version:
|
|
self._setup_virtualenv()
|
|
if not (cmd_exists('twistd') and cmd_exists('buildbot')):
|
|
raise RuntimeError('Buildbot is not installed.')
|
|
self._setup_server_workdir()
|
|
server_runner = multiprocessing.Process(target=self._start_server)
|
|
server_runner.start()
|
|
self._wait_for_server_ready()
|
|
if number_workers == 0:
|
|
print(' - To manually attach a build worker use this info:\n'
|
|
+ ' TCP port for the worker-to-server connection: {}\n'.format(self._server_pb_port)
|
|
+ ' worker-id: the one defined at {}\n'.format(os.path.join(self._server_wordir, 'passwords.json'))
|
|
+ ' password: password\n')
|
|
elif number_workers == 1:
|
|
worker = 'local-worker'
|
|
worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,))
|
|
worker_runner.start()
|
|
print(' - Worker started!.\n'
|
|
+ ' Check the log for at {}/local-worker/worker.log\n'.format(self._base_workdir_temp)
|
|
+ ' tail -f {}/local-worker/worker.log\n'.format(self._base_workdir_temp))
|
|
worker_runner.join()
|
|
else:
|
|
worker_runners = []
|
|
for worker in self._get_list_workers():
|
|
worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,))
|
|
worker_runner.start()
|
|
worker_runners.append(worker_runner)
|
|
print(' - Workers started!.\n'
|
|
+ ' Check the log for each one at {}/WORKERNAMEID/worker.log\n'.format(self._base_workdir_temp)
|
|
+ ' tail -f {}/*/worker.log\n'.format(self._base_workdir_temp))
|
|
for worker_runner in worker_runners:
|
|
worker_runner.join()
|
|
server_runner.join()
|
|
except KeyboardInterrupt:
|
|
pass # no print the exception
|
|
except:
|
|
traceback.print_exc()
|
|
finally:
|
|
try:
|
|
# The children may exit between the check and the kill call.
|
|
# Ignore any exception raised here.
|
|
for c in multiprocessing.active_children():
|
|
# Send the signal to the whole process group.
|
|
# Otherwise some twistd sub-childs can remain alive.
|
|
os.killpg(os.getpgid(c.pid), signal.SIGKILL)
|
|
except:
|
|
pass
|
|
if not no_clean:
|
|
self._clean()
|
|
sys.exit(0)
|
|
|
|
def _wait_for_server_ready(self):
|
|
server_ready_check_counter = 0
|
|
while True:
|
|
if os.path.isfile(self._server_ready_fd):
|
|
return
|
|
if server_ready_check_counter > 60:
|
|
print('ERROR: buildbot server has not started after waiting 60 seconds for it.')
|
|
|
|
if os.path.isfile(self._server_log):
|
|
print('The server.log file contains the following:')
|
|
with open(self._server_log, 'r') as f:
|
|
print(f.read())
|
|
else:
|
|
print('There is no server.log on the directory. That means a general failure starting buildbot.')
|
|
|
|
raise RuntimeError('buildbot server has not started after waiting 60 seconds for it.')
|
|
sleep(1)
|
|
server_ready_check_counter += 1
|
|
|
|
def _create_mock_worker_passwords_dict(self):
|
|
with open(os.path.join(self._server_wordir, 'config.json'), 'r') as config_json:
|
|
config_dict = json.load(config_json)
|
|
result = dict([(worker['name'], 'password') for worker in config_dict['workers']])
|
|
return result
|
|
|
|
def _setup_server_workdir(self):
|
|
self._server_wordir = os.path.join(self._base_workdir_temp, 'buildbot-server')
|
|
assert(not os.path.exists(self._server_wordir))
|
|
self._server_log = os.path.join(self._server_wordir, 'server.log')
|
|
self._server_ready_fd = os.path.join(self._server_wordir, '.server-is-ready')
|
|
print('Copying files from {} to {} ...'.format(self._configdir, self._server_wordir))
|
|
shutil.copytree(self._configdir, self._server_wordir)
|
|
print('Generating buildbot files at {} ...'.format(self._server_wordir))
|
|
with open(os.path.join(self._server_wordir, 'buildbot.tac'), 'w') as f:
|
|
f.write(buildbot_server_tac % {'config_file': self._server_config_file_name})
|
|
with open(os.path.join(self._server_wordir, 'passwords.json'), 'w') as passwords_file:
|
|
passwords_file.write(json.dumps(self._create_mock_worker_passwords_dict(), indent=4, sort_keys=True))
|
|
|
|
def _setup_virtualenv(self):
|
|
if cmd_exists('virtualenv'):
|
|
print('Setting up virtualenv at {} ... '.format(self._base_workdir_temp))
|
|
virtualenv_cmd = ['virtualenv', '-p', 'python3', 'venv']
|
|
virtualenv_process = subprocess.Popen(virtualenv_cmd, cwd=self._base_workdir_temp,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
stdout, stderr = virtualenv_process.communicate()
|
|
print_if_error_stdout_stderr(virtualenv_cmd, virtualenv_process.returncode, stdout, stderr)
|
|
virtualenv_bindir = os.path.join(self._base_workdir_temp, 'venv', 'bin')
|
|
virtualenv_pip = os.path.join(virtualenv_bindir, 'pip')
|
|
if not os.access(virtualenv_pip, os.X_OK):
|
|
print('Something went wrong setting up virtualenv'
|
|
'Trying to continue using the system version of buildbot')
|
|
return
|
|
print('Setting up buildbot dependencies on the virtualenv ... ')
|
|
# The idea is to install the very same version of buildbot and its
|
|
# dependencies than the ones used for running https://build.webkit.org/about
|
|
buildbot_version = '2.10.5'
|
|
pip_cmd = [virtualenv_pip, 'install',
|
|
'buildbot=={}'.format(buildbot_version),
|
|
'buildbot-console-view=={}'.format(buildbot_version),
|
|
'buildbot-grid-view=={}'.format(buildbot_version),
|
|
'buildbot-waterfall-view=={}'.format(buildbot_version),
|
|
'buildbot-worker=={}'.format(buildbot_version),
|
|
'buildbot-www=={}'.format(buildbot_version),
|
|
'twisted==21.2.0',
|
|
'requests==2.21.0',
|
|
'lz4==1.1.0']
|
|
pip_process = subprocess.Popen(pip_cmd, cwd=self._base_workdir_temp,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
stdout, stderr = pip_process.communicate()
|
|
print_if_error_stdout_stderr(pip_cmd, pip_process.returncode, stdout, stderr)
|
|
os.environ['PATH'] = virtualenv_bindir + ':' + os.environ['PATH']
|
|
return
|
|
print('WARNING: virtualenv not installed. '
|
|
'Trying to continue using the system version of buildbot')
|
|
|
|
def _upgrade_db_needed(self):
|
|
with open(self._server_log) as f:
|
|
for l in f:
|
|
if 'upgrade the database' in l:
|
|
return True
|
|
return False
|
|
|
|
def _start_server(self):
|
|
# This is started via multiprocessing. We set a new process group here
|
|
# to be able to reliably kill this subprocess and all of its child on clean.
|
|
os.setsid()
|
|
dbupgraded = False
|
|
retry = True
|
|
if check_tcp_port_open('localhost', self._server_pb_port):
|
|
print('ERROR: There is some process already listening in port {}'.format(self._server_pb_port))
|
|
return 1
|
|
while retry:
|
|
retry = False
|
|
print('Starting the buildbot server process ...')
|
|
twistd_cmd = ['twistd', '-l', self._server_log, '-noy', 'buildbot.tac']
|
|
twistd_server_process = subprocess.Popen(twistd_cmd, cwd=self._server_wordir,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
while twistd_server_process.poll() is None:
|
|
if check_tcp_port_open('localhost', self._server_pb_port):
|
|
print('Test buildbot server ready!\n'
|
|
+ 'Press CTRL-C to stop\n\n'
|
|
+ ' - See buildbot server log:\n'
|
|
+ ' tail -f {}\n\n'.format(self._server_log)
|
|
+ ' - Open a browser to:\n'
|
|
+ ' http://localhost:{}\n'.format(self._server_http_port))
|
|
with open(self._server_ready_fd, 'w') as f:
|
|
f.write('ready')
|
|
twistd_server_process.wait()
|
|
return 0
|
|
sleep(1)
|
|
stdout, stderr = twistd_server_process.communicate()
|
|
if twistd_server_process.returncode == 0 and self._upgrade_db_needed() and not dbupgraded:
|
|
retry = True
|
|
dbupgraded = True
|
|
print('Upgrading the database ...')
|
|
upgrade_cmd = ['buildbot', 'upgrade-master', self._server_wordir]
|
|
upgrade_process = subprocess.Popen(upgrade_cmd, cwd=self._server_wordir,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
stdout, stderr = upgrade_process.communicate()
|
|
print_if_error_stdout_stderr(upgrade_cmd, upgrade_process.returncode, stdout, stderr)
|
|
else:
|
|
print_if_error_stdout_stderr(twistd_cmd, twistd_server_process.returncode, stdout, stderr,
|
|
'Check the log at {}'.format(self._server_log))
|
|
return twistd_server_process.returncode
|
|
|
|
def _get_list_workers(self):
|
|
password_list = os.path.join(self._server_wordir, 'passwords.json')
|
|
with open(password_list) as f:
|
|
passwords = json.load(f)
|
|
list_workers = []
|
|
for worker in passwords:
|
|
list_workers.append(str(worker))
|
|
return list_workers
|
|
|
|
def _start_worker(self, worker):
|
|
# This is started via multiprocessing. We set a new process group here
|
|
# to be able to reliably kill this subprocess and all of its child on clean.
|
|
os.setsid()
|
|
worker_workdir = os.path.join(self._base_workdir_temp, worker)
|
|
os.mkdir(worker_workdir)
|
|
with open(os.path.join(worker_workdir, 'buildbot.tac'), 'w') as f:
|
|
f.write(buildbot_worker_tac % {'worker_name': worker, 'server_pb_port': self._server_pb_port})
|
|
twistd_cmd = ['twistd', '-l', 'worker.log', '-noy', 'buildbot.tac']
|
|
twistd_worker_process = subprocess.Popen(twistd_cmd, cwd=worker_workdir,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
stdout, stderr = twistd_worker_process.communicate()
|
|
print_if_error_stdout_stderr(twistd_cmd, twistd_worker_process.returncode, stdout, stderr,
|
|
'Check the log at {}'.format(os.path.join(worker_workdir, 'worker.log')))
|
|
return twistd_worker_process.returncode
|
|
|
|
def _clean(self):
|
|
if os.path.isdir(self._base_workdir_temp):
|
|
print('\n\nCleaning {} ... \n'.format(self._base_workdir_temp))
|
|
# shutil.rmtree can fail if we hold an open file descriptor on temp_dir
|
|
# (which is very likely when cleaning) or if temp_dir is a NFS mount.
|
|
# Use rm instead that always works.
|
|
rm = subprocess.Popen(['rm', '-fr', self._base_workdir_temp])
|
|
rm.wait()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if sys.version_info < (3, 5):
|
|
print('ERROR: Please use Python 3. This script is not compatible with Python 2.')
|
|
sys.exit(1)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
configuration = parser.add_mutually_exclusive_group(required=True)
|
|
configuration.add_argument('--ews', action='store_const', const='ews', dest='configuration',
|
|
help='Simulate the EWS buildbot server (ews-build.webkit.org)')
|
|
configuration.add_argument('--post-commit', action='store_const', const='post_commit', dest='configuration',
|
|
help='Simulate the post-commit buildbot server (build.webkit.org)')
|
|
configuration.add_argument('--config-dir', default=None, dest='configdir', type=str,
|
|
help='Specify the directory that contains the buildbot config files')
|
|
parser.add_argument('--base-temp-dir', help='Path where the temporal working directory will be created. '
|
|
'Note: To trigger test builds with the test workers you need enough free space on that path.',
|
|
dest='basetempdir', default=None, type=str)
|
|
parser.add_argument('--no-clean', help='Do not clean the temporal working dir on exit.',
|
|
dest='no_clean', action='store_true')
|
|
workers = parser.add_mutually_exclusive_group(required=False)
|
|
workers.add_argument('--no-workers', help='Do not start any workers.',
|
|
dest='number_workers', action='store_const', const=0)
|
|
workers.add_argument('--all-workers', help='Instead of starting only one worker that round-robins between all the queues, '
|
|
'start multiple parallel workers as defined on the server config.',
|
|
dest='number_workers', action='store_const', const=float('inf'))
|
|
parser.add_argument('--use-system-version', help='Instead of setting up a virtualenv with the buildbot version '
|
|
'used by build.webkit.org, use the buildbot version installed on this system.',
|
|
dest='use_system_version', action='store_true')
|
|
args = parser.parse_args()
|
|
|
|
if args.configuration == "ews":
|
|
configdir = os.path.join(os.path.dirname(__file__), 'ews-build')
|
|
elif args.configuration == "post_commit":
|
|
configdir = os.path.join(os.path.dirname(__file__), 'build-webkit-org')
|
|
else:
|
|
configdir = args.configdir
|
|
|
|
if args.number_workers is None:
|
|
args.number_workers = 1
|
|
|
|
buildbot_test_runner = BuildbotTestRunner(configdir)
|
|
buildbot_test_runner.start(args.basetempdir, args.no_clean, args.number_workers, args.use_system_version)
|