# Copyright (C) 2017-2021 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import operator import os import re from buildbot.scheduler import AnyBranchScheduler, Triggerable, Nightly from buildbot.schedulers.forcesched import BooleanParameter, CodebaseParameter, FixedParameter, ForceScheduler, StringParameter from buildbot.schedulers.filter import ChangeFilter from buildbot.process import buildstep, factory, properties from buildbot.util import identifiers as buildbot_identifiers from buildbot.worker import Worker from factories import * import wkbuild trunk_filter = ChangeFilter(branch=["trunk", None]) BUILDER_NAME_LENGTH_LIMIT = 70 STEP_NAME_LENGTH_LIMIT = 50 def pickLatestBuild(builder, requests): return max(requests, key=operator.attrgetter("submittedAt")) def loadBuilderConfig(c, is_test_mode_enabled=False, master_prefix_path='./'): with open(os.path.join(master_prefix_path, 'config.json')) as config_json: config = json.load(config_json) if is_test_mode_enabled: passwords = {} else: passwords = json.load(open(os.path.join(master_prefix_path, 'passwords.json'))) results_server_api_key = passwords.get('results-server-api-key') if results_server_api_key: os.environ['RESULTS_SERVER_API_KEY'] = results_server_api_key checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders']) checkValidSchedulers(config, config['schedulers']) c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=1) for worker in config['workers']] if is_test_mode_enabled: c['workers'].append(Worker('local-worker', 'password', max_builds=1)) c['schedulers'] = [] for scheduler in config['schedulers']: if "change_filter" in scheduler: scheduler["change_filter"] = globals()[scheduler["change_filter"]] schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] c['schedulers'].append(schedulerClass(**scheduler)) # Setup force schedulers builderNames = [str(builder['name']) for builder in config['builders']] reason = StringParameter(name='reason', default='', size=40) properties = [BooleanParameter(name='is_clean', label='Force Clean build')] # Disable default enabled input fields: revision, repository, project and branch codebases = [CodebaseParameter("", revision=FixedParameter(name="revision", default=""), repository=FixedParameter(name="repository", default=""), project=FixedParameter(name="project", default=""), branch=FixedParameter(name="branch", default=""))] forceScheduler = ForceScheduler(name='force', builderNames=builderNames, reason=reason, codebases=codebases, properties=properties) c['schedulers'].append(forceScheduler) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) platform = builder['platform'] factoryName = builder.pop('factory') factory = globals()[factoryName] factorykwargs = {} for key in ['platform', 'configuration', 'architectures', 'triggers', 'additionalArguments', 'device_model']: value = builder.pop(key, None) if value: factorykwargs[key] = value builder['factory'] = factory(**factorykwargs) if is_test_mode_enabled: builder['workernames'].append('local-worker') builder_name = builder['name'] for step in builder["factory"].steps: step_name = step.buildStep().name if len(step_name) > STEP_NAME_LENGTH_LIMIT: raise Exception('step name "{}" is longer than maximum allowed by Buildbot ({} characters).'.format(step_name, STEP_NAME_LENGTH_LIMIT)) if not buildbot_identifiers.ident_re.match(step_name): raise Exception('step name "{}" is not a valid buildbot identifier.'.format(step_name)) if platform.startswith('mac'): category = 'AppleMac' elif platform.startswith('ios'): category = 'iOS' elif platform == 'win': category = 'AppleWin' elif platform.startswith('gtk'): category = 'GTK' elif platform.startswith('wpe'): category = 'WPE' elif platform == 'wincairo': category = 'WinCairo' elif platform.startswith('playstation'): category = 'PlayStation' else: category = 'misc' if (category in ('AppleMac', 'AppleWin', 'iOS')) and factoryName != 'BuildFactory': builder['nextBuild'] = pickLatestBuild c['builders'].append(builder) class PlatformSpecificScheduler(AnyBranchScheduler): def __init__(self, platform, branch, **kwargs): self.platform = platform filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter) AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs) def filter(self, change): return wkbuild.should_build(self.platform, change.files) def checkValidWorker(worker): if not worker: raise Exception('Worker is None or Empty.') if not worker.get('name'): raise Exception('Worker "{}" does not have name defined.'.format(worker)) if not worker.get('platform'): raise Exception('Worker {} does not have platform defined.'.format(worker['name'])) def checkValidBuilder(config, builder): if not builder: raise Exception('Builder is None or Empty.') if not builder.get('name'): raise Exception('Builder "{}" does not have name defined.'.format(builder)) if not buildbot_identifiers.ident_re.match(builder['name']): raise Exception('Builder name {} is not a valid buildbot identifier.'.format(builder['name'])) if len(builder['name']) > BUILDER_NAME_LENGTH_LIMIT: raise Exception('Builder name {} is longer than maximum allowed by Buildbot ({} characters).'.format(builder['name'], BUILDER_NAME_LENGTH_LIMIT)) if 'configuration' in builder and builder['configuration'] not in ['debug', 'production', 'release']: raise Exception('Invalid configuration: {} for builder: {}'.format(builder.get('configuration'), builder.get('name'))) if not builder.get('factory'): raise Exception('Builder {} does not have factory defined.'.format(builder['name'])) if not builder.get('platform'): raise Exception('Builder {} does not have platform defined.'.format(builder['name'])) for trigger in builder.get('triggers') or []: if not doesTriggerExist(config, trigger): raise Exception('Trigger: {} in builder {} does not exist in list of Trigerrable schedulers.'.format(trigger, builder['name'])) def checkValidSchedulers(config, schedulers): for scheduler in config.get('schedulers') or []: if scheduler.get('type') == 'Triggerable': if not isTriggerUsedByAnyBuilder(config, scheduler['name']) and 'build' not in scheduler['name'].lower(): raise Exception('Trigger: {} is not used by any builder in config.json'.format(scheduler['name'])) def doesTriggerExist(config, trigger): for scheduler in config.get('schedulers') or []: if scheduler.get('name') == trigger: return True return False def isTriggerUsedByAnyBuilder(config, trigger): for builder in config.get('builders'): if trigger in (builder.get('triggers') or []): return True return False def checkWorkersAndBuildersForConsistency(config, workers, builders): def _find_worker_with_name(workers, worker_name): result = None for worker in workers: if worker['name'] == worker_name: if not result: result = worker else: raise Exception('Duplicate worker entry found for {}.'.format(worker['name'])) return result for worker in workers: checkValidWorker(worker) for builder in builders: checkValidBuilder(config, builder) for worker_name in builder['workernames']: worker = _find_worker_with_name(workers, worker_name) if worker is None: raise Exception('Builder {} has worker {}, which is not defined in workers list!'.format(builder['name'], worker_name)) if worker['platform'] != builder['platform'] and worker['platform'] != '*' and builder['platform'] != '*': raise Exception('Builder "{0}" is for platform "{1}", but has worker "{2}" for platform "{3}"!'.format( builder['name'], builder['platform'], worker['name'], worker['platform'])) def getInvalidTags(): """ We maintain a list of words which we do not want to display as tag in buildbot. We generate a list of tags by splitting the builder name. We do not want certain words as tag. For e.g. we don't want '11'as tag for builder iOS-11-Simulator-EWS """ invalid_tags = [str(i) for i in range(0, 20)] invalid_tags.extend(['EWS', 'TryBot']) return invalid_tags def getValidTags(tags): return list(set(tags) - set(getInvalidTags())) def getTagsForBuilder(builder): keywords = re.split(r'[, \-_:()]+', str(builder['name'])) return getValidTags(keywords)