249 lines
10 KiB
Python
249 lines
10 KiB
Python
# Copyright (C) 2017-2021 Apple Inc. All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
|
|
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
|
|
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
import json
|
|
import operator
|
|
import os
|
|
import re
|
|
|
|
from buildbot.scheduler import AnyBranchScheduler, Triggerable, Nightly
|
|
from buildbot.schedulers.forcesched import BooleanParameter, CodebaseParameter, FixedParameter, ForceScheduler, StringParameter
|
|
from buildbot.schedulers.filter import ChangeFilter
|
|
from buildbot.process import buildstep, factory, properties
|
|
from buildbot.util import identifiers as buildbot_identifiers
|
|
from buildbot.worker import Worker
|
|
|
|
from factories import *
|
|
import wkbuild
|
|
|
|
trunk_filter = ChangeFilter(branch=["trunk", None])
|
|
|
|
BUILDER_NAME_LENGTH_LIMIT = 70
|
|
STEP_NAME_LENGTH_LIMIT = 50
|
|
|
|
|
|
def pickLatestBuild(builder, requests):
|
|
return max(requests, key=operator.attrgetter("submittedAt"))
|
|
|
|
|
|
def loadBuilderConfig(c, is_test_mode_enabled=False, master_prefix_path='./'):
|
|
with open(os.path.join(master_prefix_path, 'config.json')) as config_json:
|
|
config = json.load(config_json)
|
|
if is_test_mode_enabled:
|
|
passwords = {}
|
|
else:
|
|
passwords = json.load(open(os.path.join(master_prefix_path, 'passwords.json')))
|
|
results_server_api_key = passwords.get('results-server-api-key')
|
|
if results_server_api_key:
|
|
os.environ['RESULTS_SERVER_API_KEY'] = results_server_api_key
|
|
|
|
checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders'])
|
|
checkValidSchedulers(config, config['schedulers'])
|
|
|
|
c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=1) for worker in config['workers']]
|
|
if is_test_mode_enabled:
|
|
c['workers'].append(Worker('local-worker', 'password', max_builds=1))
|
|
|
|
c['schedulers'] = []
|
|
for scheduler in config['schedulers']:
|
|
if "change_filter" in scheduler:
|
|
scheduler["change_filter"] = globals()[scheduler["change_filter"]]
|
|
schedulerClassName = scheduler.pop('type')
|
|
schedulerClass = globals()[schedulerClassName]
|
|
c['schedulers'].append(schedulerClass(**scheduler))
|
|
|
|
# Setup force schedulers
|
|
builderNames = [str(builder['name']) for builder in config['builders']]
|
|
reason = StringParameter(name='reason', default='', size=40)
|
|
properties = [BooleanParameter(name='is_clean', label='Force Clean build')]
|
|
# Disable default enabled input fields: revision, repository, project and branch
|
|
codebases = [CodebaseParameter("",
|
|
revision=FixedParameter(name="revision", default=""),
|
|
repository=FixedParameter(name="repository", default=""),
|
|
project=FixedParameter(name="project", default=""),
|
|
branch=FixedParameter(name="branch", default=""))]
|
|
forceScheduler = ForceScheduler(name='force', builderNames=builderNames, reason=reason, codebases=codebases, properties=properties)
|
|
c['schedulers'].append(forceScheduler)
|
|
|
|
c['builders'] = []
|
|
for builder in config['builders']:
|
|
builder['tags'] = getTagsForBuilder(builder)
|
|
platform = builder['platform']
|
|
factoryName = builder.pop('factory')
|
|
factory = globals()[factoryName]
|
|
factorykwargs = {}
|
|
for key in ['platform', 'configuration', 'architectures', 'triggers', 'additionalArguments', 'device_model']:
|
|
value = builder.pop(key, None)
|
|
if value:
|
|
factorykwargs[key] = value
|
|
|
|
builder['factory'] = factory(**factorykwargs)
|
|
|
|
if is_test_mode_enabled:
|
|
builder['workernames'].append('local-worker')
|
|
|
|
builder_name = builder['name']
|
|
for step in builder["factory"].steps:
|
|
step_name = step.buildStep().name
|
|
if len(step_name) > STEP_NAME_LENGTH_LIMIT:
|
|
raise Exception('step name "{}" is longer than maximum allowed by Buildbot ({} characters).'.format(step_name, STEP_NAME_LENGTH_LIMIT))
|
|
if not buildbot_identifiers.ident_re.match(step_name):
|
|
raise Exception('step name "{}" is not a valid buildbot identifier.'.format(step_name))
|
|
|
|
if platform.startswith('mac'):
|
|
category = 'AppleMac'
|
|
elif platform.startswith('ios'):
|
|
category = 'iOS'
|
|
elif platform == 'win':
|
|
category = 'AppleWin'
|
|
elif platform.startswith('gtk'):
|
|
category = 'GTK'
|
|
elif platform.startswith('wpe'):
|
|
category = 'WPE'
|
|
elif platform == 'wincairo':
|
|
category = 'WinCairo'
|
|
elif platform.startswith('playstation'):
|
|
category = 'PlayStation'
|
|
else:
|
|
category = 'misc'
|
|
|
|
if (category in ('AppleMac', 'AppleWin', 'iOS')) and factoryName != 'BuildFactory':
|
|
builder['nextBuild'] = pickLatestBuild
|
|
|
|
c['builders'].append(builder)
|
|
|
|
|
|
class PlatformSpecificScheduler(AnyBranchScheduler):
|
|
def __init__(self, platform, branch, **kwargs):
|
|
self.platform = platform
|
|
filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
|
|
AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
|
|
|
|
def filter(self, change):
|
|
return wkbuild.should_build(self.platform, change.files)
|
|
|
|
|
|
def checkValidWorker(worker):
|
|
if not worker:
|
|
raise Exception('Worker is None or Empty.')
|
|
|
|
if not worker.get('name'):
|
|
raise Exception('Worker "{}" does not have name defined.'.format(worker))
|
|
|
|
if not worker.get('platform'):
|
|
raise Exception('Worker {} does not have platform defined.'.format(worker['name']))
|
|
|
|
|
|
def checkValidBuilder(config, builder):
|
|
if not builder:
|
|
raise Exception('Builder is None or Empty.')
|
|
|
|
if not builder.get('name'):
|
|
raise Exception('Builder "{}" does not have name defined.'.format(builder))
|
|
|
|
if not buildbot_identifiers.ident_re.match(builder['name']):
|
|
raise Exception('Builder name {} is not a valid buildbot identifier.'.format(builder['name']))
|
|
|
|
if len(builder['name']) > BUILDER_NAME_LENGTH_LIMIT:
|
|
raise Exception('Builder name {} is longer than maximum allowed by Buildbot ({} characters).'.format(builder['name'], BUILDER_NAME_LENGTH_LIMIT))
|
|
|
|
if 'configuration' in builder and builder['configuration'] not in ['debug', 'production', 'release']:
|
|
raise Exception('Invalid configuration: {} for builder: {}'.format(builder.get('configuration'), builder.get('name')))
|
|
|
|
if not builder.get('factory'):
|
|
raise Exception('Builder {} does not have factory defined.'.format(builder['name']))
|
|
|
|
if not builder.get('platform'):
|
|
raise Exception('Builder {} does not have platform defined.'.format(builder['name']))
|
|
|
|
for trigger in builder.get('triggers') or []:
|
|
if not doesTriggerExist(config, trigger):
|
|
raise Exception('Trigger: {} in builder {} does not exist in list of Trigerrable schedulers.'.format(trigger, builder['name']))
|
|
|
|
|
|
def checkValidSchedulers(config, schedulers):
|
|
for scheduler in config.get('schedulers') or []:
|
|
if scheduler.get('type') == 'Triggerable':
|
|
if not isTriggerUsedByAnyBuilder(config, scheduler['name']) and 'build' not in scheduler['name'].lower():
|
|
raise Exception('Trigger: {} is not used by any builder in config.json'.format(scheduler['name']))
|
|
|
|
|
|
def doesTriggerExist(config, trigger):
|
|
for scheduler in config.get('schedulers') or []:
|
|
if scheduler.get('name') == trigger:
|
|
return True
|
|
return False
|
|
|
|
|
|
def isTriggerUsedByAnyBuilder(config, trigger):
|
|
for builder in config.get('builders'):
|
|
if trigger in (builder.get('triggers') or []):
|
|
return True
|
|
return False
|
|
|
|
|
|
def checkWorkersAndBuildersForConsistency(config, workers, builders):
|
|
def _find_worker_with_name(workers, worker_name):
|
|
result = None
|
|
for worker in workers:
|
|
if worker['name'] == worker_name:
|
|
if not result:
|
|
result = worker
|
|
else:
|
|
raise Exception('Duplicate worker entry found for {}.'.format(worker['name']))
|
|
return result
|
|
|
|
for worker in workers:
|
|
checkValidWorker(worker)
|
|
|
|
for builder in builders:
|
|
checkValidBuilder(config, builder)
|
|
for worker_name in builder['workernames']:
|
|
worker = _find_worker_with_name(workers, worker_name)
|
|
if worker is None:
|
|
raise Exception('Builder {} has worker {}, which is not defined in workers list!'.format(builder['name'], worker_name))
|
|
|
|
if worker['platform'] != builder['platform'] and worker['platform'] != '*' and builder['platform'] != '*':
|
|
raise Exception('Builder "{0}" is for platform "{1}", but has worker "{2}" for platform "{3}"!'.format(
|
|
builder['name'], builder['platform'], worker['name'], worker['platform']))
|
|
|
|
|
|
def getInvalidTags():
|
|
"""
|
|
We maintain a list of words which we do not want to display as tag in buildbot.
|
|
We generate a list of tags by splitting the builder name. We do not want certain words as tag.
|
|
For e.g. we don't want '11'as tag for builder iOS-11-Simulator-EWS
|
|
"""
|
|
invalid_tags = [str(i) for i in range(0, 20)]
|
|
invalid_tags.extend(['EWS', 'TryBot'])
|
|
return invalid_tags
|
|
|
|
|
|
def getValidTags(tags):
|
|
return list(set(tags) - set(getInvalidTags()))
|
|
|
|
|
|
def getTagsForBuilder(builder):
|
|
keywords = re.split(r'[, \-_:()]+', str(builder['name']))
|
|
return getValidTags(keywords)
|