2016-06-08 20:43:01 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const assert = require('assert');
|
|
|
|
|
|
|
|
require('../tools/js/v3-models.js');
|
|
|
|
|
|
|
|
const TestServer = require('./resources/test-server.js');
|
|
|
|
const MockData = require('./resources/mock-data.js');
|
2021-03-22 21:23:38 +00:00
|
|
|
const addWorkerForReport = require('./resources/common-operations.js').addWorkerForReport;
|
2017-03-15 03:19:02 +00:00
|
|
|
const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
|
2016-06-08 20:43:01 +00:00
|
|
|
|
|
|
|
describe('/api/update-triggerable/', function () {
|
2017-03-15 03:19:02 +00:00
|
|
|
prepareServerTest(this);
|
2016-06-08 20:43:01 +00:00
|
|
|
|
|
|
|
const emptyUpdate = {
|
2021-03-22 21:23:38 +00:00
|
|
|
'workerName': 'someWorker',
|
|
|
|
'workerPassword': 'somePassword',
|
2016-06-08 20:43:01 +00:00
|
|
|
'triggerable': 'build-webkit',
|
|
|
|
'configurations': [],
|
|
|
|
};
|
|
|
|
|
|
|
|
const smallUpdate = {
|
2021-03-22 21:23:38 +00:00
|
|
|
'workerName': 'someWorker',
|
|
|
|
'workerPassword': 'somePassword',
|
2016-06-08 20:43:01 +00:00
|
|
|
'triggerable': 'build-webkit',
|
|
|
|
'configurations': [
|
|
|
|
{test: MockData.someTestId(), platform: MockData.somePlatformId()}
|
|
|
|
],
|
|
|
|
};
|
|
|
|
|
2021-03-22 21:23:38 +00:00
|
|
|
it('should reject when worker name is missing', () => {
|
2017-03-15 03:19:02 +00:00
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', {}).then((response) => {
|
2021-03-22 21:23:38 +00:00
|
|
|
assert.strictEqual(response['status'], 'MissingWorkerName');
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
2021-03-22 21:23:38 +00:00
|
|
|
it('should reject when there are no workers', () => {
|
|
|
|
const update = {workerName: emptyUpdate.workerName, workerPassword: emptyUpdate.workerPassword};
|
2017-03-15 03:19:02 +00:00
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update).then((response) => {
|
2021-03-22 21:23:38 +00:00
|
|
|
assert.strictEqual(response['status'], 'WorkerNotFound');
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
2021-03-22 21:23:38 +00:00
|
|
|
it('should reject when the worker password doesn\'t match', () => {
|
2017-03-15 03:19:02 +00:00
|
|
|
return MockData.addMockData(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(emptyUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
const report = {workerName: emptyUpdate.workerName, workerPassword: 'badPassword'};
|
2016-06-08 20:43:01 +00:00
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', emptyUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'OK');
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
2017-03-15 03:19:02 +00:00
|
|
|
it('should accept an empty report', () => {
|
|
|
|
return MockData.addMockData(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(emptyUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then(() => {
|
2016-06-08 20:43:01 +00:00
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', emptyUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'OK');
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
2017-03-15 03:19:02 +00:00
|
|
|
it('delete existing configurations when accepting an empty report', () => {
|
2016-06-08 20:43:01 +00:00
|
|
|
const db = TestServer.database();
|
2017-03-15 03:19:02 +00:00
|
|
|
return MockData.addMockData(db).then(() => {
|
2016-06-08 20:43:01 +00:00
|
|
|
return Promise.all([
|
2021-03-22 21:23:38 +00:00
|
|
|
addWorkerForReport(emptyUpdate),
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
db.insert('triggerable_configurations', {'triggerable': 1000 // build-webkit
|
|
|
|
, 'test': MockData.someTestId(), 'platform': MockData.somePlatformId()})
|
2016-06-08 20:43:01 +00:00
|
|
|
]);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then(() => {
|
2016-06-08 20:43:01 +00:00
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', emptyUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'OK');
|
2016-06-08 20:43:01 +00:00
|
|
|
return db.selectAll('triggerable_configurations', 'test');
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((rows) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(rows.length, 0);
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
2017-03-15 03:19:02 +00:00
|
|
|
it('should add configurations in the update', () => {
|
2016-06-08 20:43:01 +00:00
|
|
|
const db = TestServer.database();
|
2017-03-15 03:19:02 +00:00
|
|
|
return MockData.addMockData(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(smallUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then(() => {
|
2016-06-08 20:43:01 +00:00
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', smallUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'OK');
|
2016-06-08 20:43:01 +00:00
|
|
|
return db.selectAll('triggerable_configurations', 'test');
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((rows) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(rows.length, 1);
|
|
|
|
assert.strictEqual(rows[0]['test'], smallUpdate.configurations[0]['test']);
|
|
|
|
assert.strictEqual(rows[0]['platform'], smallUpdate.configurations[0]['platform']);
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
2017-03-15 03:19:02 +00:00
|
|
|
it('should reject when a configuration is malformed', () => {
|
|
|
|
return MockData.addMockData(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(smallUpdate);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then(() => {
|
2016-06-08 20:43:01 +00:00
|
|
|
const update = {
|
2021-03-22 21:23:38 +00:00
|
|
|
'workerName': 'someWorker',
|
|
|
|
'workerPassword': 'somePassword',
|
2016-06-08 20:43:01 +00:00
|
|
|
'triggerable': 'build-webkit',
|
|
|
|
'configurations': [{}],
|
|
|
|
};
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
2017-03-15 03:19:02 +00:00
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidConfigurationEntry');
|
2017-03-15 03:19:02 +00:00
|
|
|
});
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|
|
|
|
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
function updateWithOSXRepositoryGroup()
|
|
|
|
{
|
|
|
|
return {
|
2021-03-22 21:23:38 +00:00
|
|
|
'workerName': 'someWorker',
|
|
|
|
'workerPassword': 'somePassword',
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
'triggerable': 'empty-triggerable',
|
|
|
|
'configurations': [
|
|
|
|
{test: MockData.someTestId(), platform: MockData.somePlatformId()}
|
|
|
|
],
|
|
|
|
'repositoryGroups': [
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
{name: 'system-only', repositories: [
|
|
|
|
{repository: MockData.macosRepositoryId(), acceptsPatch: false},
|
|
|
|
]},
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
]
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
it('should reject when repositoryGroups is not an array', () => {
|
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
update.repositoryGroups = 1;
|
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidRepositoryGroups');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should reject when the name of a repository group is not specified', () => {
|
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
delete update.repositoryGroups[0].name;
|
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidRepositoryGroup');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should reject when the repository list is not specified for a repository group', () => {
|
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
delete update.repositoryGroups[0].repositories;
|
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidRepositoryGroup');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should reject when the repository list of a repository group is not an array', () => {
|
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
update.repositoryGroups[0].repositories = 'hi';
|
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidRepositoryGroup');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
it('should reject when a repository group contains a repository data that is not an array', () => {
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
update.repositoryGroups[0].repositories[0] = 999;
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidRepositoryData');
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should reject when a repository group contains an invalid repository id', () => {
|
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
update.repositoryGroups[0].repositories[0] = {repository: 999};
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'InvalidRepository');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should reject when a repository group contains a duplicate repository id', () => {
|
|
|
|
const update = updateWithOSXRepositoryGroup();
|
|
|
|
const group = update.repositoryGroups[0];
|
|
|
|
group.repositories.push(group.repositories[0]);
|
|
|
|
return MockData.addEmptyTriggerable(TestServer.database()).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(update);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', update);
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'DuplicateRepository');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should add a new repository group when there are none', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(updateWithOSXRepositoryGroup());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSON('/api/update-triggerable/', updateWithOSXRepositoryGroup());
|
|
|
|
}).then((response) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(response['status'], 'OK');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
return Promise.all([db.selectAll('triggerable_configurations', 'test'), db.selectAll('triggerable_repository_groups')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [configurations, repositoryGroups] = result;
|
|
|
|
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(configurations.length, 1);
|
|
|
|
assert.strictEqual(configurations[0]['test'], MockData.someTestId());
|
|
|
|
assert.strictEqual(configurations[0]['platform'], MockData.somePlatformId());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 1);
|
|
|
|
assert.strictEqual(repositoryGroups[0]['name'], 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[0]['triggerable'], MockData.emptyTriggeragbleId());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should not add a duplicate repository group when there is a group of the same name', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
let initialResult;
|
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(updateWithOSXRepositoryGroup());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', updateWithOSXRepositoryGroup());
|
|
|
|
}).then((response) => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_configurations', 'test'), db.selectAll('triggerable_repository_groups')]);
|
|
|
|
}).then((result) => {
|
|
|
|
initialResult = result;
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', updateWithOSXRepositoryGroup());
|
|
|
|
}).then(() => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_configurations', 'test'), db.selectAll('triggerable_repository_groups')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [initialConfigurations, initialRepositoryGroups] = initialResult;
|
|
|
|
const [configurations, repositoryGroups] = result;
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.deepStrictEqual(configurations, initialConfigurations);
|
|
|
|
assert.deepStrictEqual(repositoryGroups, initialRepositoryGroups);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
})
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should not add a duplicate repository group when there is a group of the same name', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
let initialResult;
|
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(updateWithOSXRepositoryGroup());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', updateWithOSXRepositoryGroup());
|
|
|
|
}).then((response) => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_configurations', 'test'), db.selectAll('triggerable_repository_groups')]);
|
|
|
|
}).then((result) => {
|
|
|
|
initialResult = result;
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', updateWithOSXRepositoryGroup());
|
|
|
|
}).then(() => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_configurations', 'test'), db.selectAll('triggerable_repository_groups')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [initialConfigurations, initialRepositoryGroups] = initialResult;
|
|
|
|
const [configurations, repositoryGroups] = result;
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.deepStrictEqual(configurations, initialConfigurations);
|
|
|
|
assert.deepStrictEqual(repositoryGroups, initialRepositoryGroups);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
})
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should update the description of a repository group when the name matches', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
const initialUpdate = updateWithOSXRepositoryGroup();
|
|
|
|
const secondUpdate = updateWithOSXRepositoryGroup();
|
|
|
|
secondUpdate.repositoryGroups[0].description = 'this group is awesome';
|
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(initialUpdate);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', initialUpdate);
|
|
|
|
}).then((response) => db.selectAll('triggerable_repository_groups')).then((repositoryGroups) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 1);
|
|
|
|
assert.strictEqual(repositoryGroups[0]['name'], 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[0]['description'], null);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', secondUpdate);
|
|
|
|
}).then(() => db.selectAll('triggerable_repository_groups')).then((repositoryGroups) => {
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 1);
|
|
|
|
assert.strictEqual(repositoryGroups[0]['name'], 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[0]['description'], 'this group is awesome');
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
function updateWithMacWebKitRepositoryGroups()
|
|
|
|
{
|
|
|
|
return {
|
2021-03-22 21:23:38 +00:00
|
|
|
'workerName': 'someWorker',
|
|
|
|
'workerPassword': 'somePassword',
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
'triggerable': 'empty-triggerable',
|
|
|
|
'configurations': [
|
|
|
|
{test: MockData.someTestId(), platform: MockData.somePlatformId()}
|
|
|
|
],
|
|
|
|
'repositoryGroups': [
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
{name: 'system-only', repositories: [{repository: MockData.macosRepositoryId()}]},
|
|
|
|
{name: 'system-and-webkit', repositories:
|
|
|
|
[{repository: MockData.webkitRepositoryId()}, {repository: MockData.macosRepositoryId()}]},
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
]
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
function mapRepositoriesByGroup(repositories)
|
|
|
|
{
|
|
|
|
const map = {};
|
|
|
|
for (const row of repositories) {
|
|
|
|
const groupId = row['group'];
|
|
|
|
if (!(groupId in map))
|
|
|
|
map[groupId] = [];
|
|
|
|
map[groupId].push(row['repository']);
|
|
|
|
}
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
it('should update the acceptable of custom roots and patches', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
const initialUpdate = updateWithMacWebKitRepositoryGroups();
|
|
|
|
const secondUpdate = updateWithMacWebKitRepositoryGroups();
|
|
|
|
secondUpdate.repositoryGroups[0].acceptsRoots = true;
|
|
|
|
secondUpdate.repositoryGroups[1].repositories[0].acceptsPatch = true;
|
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(initialUpdate);
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', initialUpdate);
|
Add the support for perf try bots to sync-buildbot.js
https://bugs.webkit.org/show_bug.cgi?id=172529
Rubber-stamped by Chris Dumez.
Make sync-buildbot.js schedule an A/B testing job with a patch or roots to buildbot.
Change the buildbot property format in the syncing script's configuration again to use a dictionary
with a single key of "revision", "patch", or "roots" to specify a revision, a patch, or a set of roots,
and simplified the structure of the configuration by always having "types" and "builders", and
make each entry in "configurations" refer to a list of types, platforms, and builders.
Since now there are build requests to build patches and run tests, "configurations" has been renamed to
"testConfigurations" and "buildConfigurations" have been added. Each entry in "buildConfigurations"
specifies a list of platforms and builders. Similarly in repository group configurations, the buildbot
properties for testing is now specified as "testProperties" and ones for building a patch is specified
in newly introduced "buildProperties".
* public/api/build-requests.php:
(update_builds): When a build request to build a patch fails, mark all subsequent requests as failed
since there is no way to run tests without a successful build.
* public/api/update-triggerable.php:
(main): Re-generate manifest.json after updating the triggerable. The lack of this re-generation was
the reason we had to manually GET /api/manifest in api-update-triggerable-tests.js.
* public/v3/models/build-request.js:
(BuildRequest.prototype.hasCompleted): Added.
* public/v3/models/manifest.js:
(Manifest.reset): Added. Extracted from MockData.resetV3Models in unit-tests/mock-data.js and
syncLoop in tools/sync-buildbot.js
(Manifest.fetch): Reset V3 models before fetching the manifest. This eliminates the need to manually
reset V3 models in syncLoop.
* public/v3/models/uploaded-file.js:
(UploadedFile.prototype.url): Use RemoteAPI.url to get the full URL instead of just a path.
* public/v3/remote.js:
(BrowserRemoteAPI.prototype.url): Added. Constructs the full URL.
* server-tests/api-update-triggerable-tests.js:
(.refetchManifest): Deleted. Now that /api/manifest re-generates manifest.json, we can simply call
Manifest.fetch instead.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Calls Manifest.reset().
(MockData.addMockConfiguration): Extracted from addMockData.
(MockData.addMockData): Updated per the format change.
(MockData.mockTestSyncConfigWithSingleBuilder): Ditto.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
(MockData.runningBuild): Make buildNumber specifiable.
(MockData.finishedBuild): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated configurations per the format change.
Now that now acceptsCustomRoots() for "system-and-webkit" must be true since we can't have a
repository group that which accepts a patch and not take roots.
* server-tests/tools-sync-buildbot-integration-tests.js: Added.
(createTriggerable): Added.
(createTestGroupWihPatch): Added.
(uploadRoot): Added.
(.assertAndResolveRequest): Added.
(.assertTestBuildHasFailed): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Added. _type as an instance variable to identify whether this buildbot builder
is a "builder" which builds a patch, builder, or a "tester" which runs a test. Also renamed
_testConfigurations to _configurations.
(BuildbotSyncer.prototype.addTestConfiguration): Assert that either the type of this syncer hasn't
been set or it's a tester.
(BuildbotSyncer.prototype.testConfigurations): Return [] when it's a builder.
(BuildbotSyncer.prototype.addBuildConfiguration): Added. Adds a platform to a builder.
(BuildbotSyncer.prototype.buildConfigurations): Added. Returns the list of configurations if this
syncer is a builder. Otherwise returns [].
(BuildbotSyncer.prototype.isTester): Added.
(BuildbotSyncer.prototype.matchesConfiguration):
(BuildbotSyncer.prototype._propertiesForBuildRequest): Updated to support the new format.
(BuildbotSyncer._loadConfig): Ditto. Optionally parse buildConfigurations.
(BuildbotSyncer._resolveBuildersWithPlatforms): Added. For each test or build configuration entry,
creates the list of configurations per builder and platform.
(BuildbotSyncer._parseRepositoryGroup): Added the support for parsing the new format with revision,
roots, and patch option types with a lot of validations as we're seeing a bit of combinatorial
explosion in the number of things that can go wrong. Also parse buildProperties optionally.
(BuildbotSyncer._parseRepositoryGroupPropertyTemplate): Added. A helper function to parse a set of
buildbot properties, validates its content, and invokes a callback if it's an dynamically resolved
type such as "revision" and "patch".
(BuildbotSyncer._validateAndMergeConfig): Updated per the format change. No longer allows "types",
"type", "platforms", and "platform" as they're explicity resolved in _resolveBuildersWithPlatforms.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.syncOnce):
(BuildbotTriggerable.prototype._validateRequests): Handle the case when a build request is not
associated with any test.
(BuildbotTriggerable.prototype._nextRequestInGroup): Return null when there is a build request to
build a patch which has not been completed (pending, scheduled, running, or failed). Since all
requests to build a patch has a negative order, those requests should all show up at the beginning.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Pick a new buildbot syncer when
scheduling the first request to build a patch or the first request to run a test. The first request
to run a test will always have order of 0, so it's a sufficient condition to find such a request.
On the other hand, the first request to build a patch can have a negative order number so we must
explicitly check if it's the first item in the ordered list of requests in the test group.
* tools/remote-server-relay.log: Added.
* tools/sync-buildbot.js:
(syncLoop): Fixed a bug we were not re-fetching the triggerable after updating the triggerable so
that Triggerable and related objects we have in the memory may not reflect what we just synced to
the perf dashboard. Also, we don't reset V3 models manually any more since Manifest.fetch does that.
* unit-tests/buildbot-syncer-tests.js: Added more test cases and updated existing test cases to test
exception messages explicitly since allowing any exception was resulting in some tests passing a
result of unrelated parsing error being thrown, etc...
(sampleiOSConfig): Updated per the format change.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto.
Canonical link: https://commits.webkit.org/189454@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@217378 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-05-24 18:47:01 +00:00
|
|
|
}).then(() => Manifest.fetch()).then(() => {
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
const repositoryGroups = TriggerableRepositoryGroup.sortByName(TriggerableRepositoryGroup.all());
|
|
|
|
const webkit = Repository.findTopLevelByName('WebKit');
|
|
|
|
const macos = Repository.findTopLevelByName('macOS');
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 2);
|
|
|
|
assert.strictEqual(repositoryGroups[0].name(), 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[0].description(), 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[0].acceptsCustomRoots(), false);
|
|
|
|
assert.deepStrictEqual(repositoryGroups[0].repositories(), [webkit, macos]);
|
|
|
|
assert.strictEqual(repositoryGroups[0].acceptsPatchForRepository(webkit), false);
|
|
|
|
assert.strictEqual(repositoryGroups[0].acceptsPatchForRepository(macos), false);
|
|
|
|
|
|
|
|
assert.strictEqual(repositoryGroups[1].name(), 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[1].description(), 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[1].acceptsCustomRoots(), false);
|
|
|
|
assert.deepStrictEqual(repositoryGroups[1].repositories(), [macos]);
|
|
|
|
assert.strictEqual(repositoryGroups[1].acceptsPatchForRepository(webkit), false);
|
|
|
|
assert.strictEqual(repositoryGroups[1].acceptsPatchForRepository(macos), false);
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', secondUpdate);
|
Add the support for perf try bots to sync-buildbot.js
https://bugs.webkit.org/show_bug.cgi?id=172529
Rubber-stamped by Chris Dumez.
Make sync-buildbot.js schedule an A/B testing job with a patch or roots to buildbot.
Change the buildbot property format in the syncing script's configuration again to use a dictionary
with a single key of "revision", "patch", or "roots" to specify a revision, a patch, or a set of roots,
and simplified the structure of the configuration by always having "types" and "builders", and
make each entry in "configurations" refer to a list of types, platforms, and builders.
Since now there are build requests to build patches and run tests, "configurations" has been renamed to
"testConfigurations" and "buildConfigurations" have been added. Each entry in "buildConfigurations"
specifies a list of platforms and builders. Similarly in repository group configurations, the buildbot
properties for testing is now specified as "testProperties" and ones for building a patch is specified
in newly introduced "buildProperties".
* public/api/build-requests.php:
(update_builds): When a build request to build a patch fails, mark all subsequent requests as failed
since there is no way to run tests without a successful build.
* public/api/update-triggerable.php:
(main): Re-generate manifest.json after updating the triggerable. The lack of this re-generation was
the reason we had to manually GET /api/manifest in api-update-triggerable-tests.js.
* public/v3/models/build-request.js:
(BuildRequest.prototype.hasCompleted): Added.
* public/v3/models/manifest.js:
(Manifest.reset): Added. Extracted from MockData.resetV3Models in unit-tests/mock-data.js and
syncLoop in tools/sync-buildbot.js
(Manifest.fetch): Reset V3 models before fetching the manifest. This eliminates the need to manually
reset V3 models in syncLoop.
* public/v3/models/uploaded-file.js:
(UploadedFile.prototype.url): Use RemoteAPI.url to get the full URL instead of just a path.
* public/v3/remote.js:
(BrowserRemoteAPI.prototype.url): Added. Constructs the full URL.
* server-tests/api-update-triggerable-tests.js:
(.refetchManifest): Deleted. Now that /api/manifest re-generates manifest.json, we can simply call
Manifest.fetch instead.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Calls Manifest.reset().
(MockData.addMockConfiguration): Extracted from addMockData.
(MockData.addMockData): Updated per the format change.
(MockData.mockTestSyncConfigWithSingleBuilder): Ditto.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
(MockData.runningBuild): Make buildNumber specifiable.
(MockData.finishedBuild): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated configurations per the format change.
Now that now acceptsCustomRoots() for "system-and-webkit" must be true since we can't have a
repository group that which accepts a patch and not take roots.
* server-tests/tools-sync-buildbot-integration-tests.js: Added.
(createTriggerable): Added.
(createTestGroupWihPatch): Added.
(uploadRoot): Added.
(.assertAndResolveRequest): Added.
(.assertTestBuildHasFailed): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Added. _type as an instance variable to identify whether this buildbot builder
is a "builder" which builds a patch, builder, or a "tester" which runs a test. Also renamed
_testConfigurations to _configurations.
(BuildbotSyncer.prototype.addTestConfiguration): Assert that either the type of this syncer hasn't
been set or it's a tester.
(BuildbotSyncer.prototype.testConfigurations): Return [] when it's a builder.
(BuildbotSyncer.prototype.addBuildConfiguration): Added. Adds a platform to a builder.
(BuildbotSyncer.prototype.buildConfigurations): Added. Returns the list of configurations if this
syncer is a builder. Otherwise returns [].
(BuildbotSyncer.prototype.isTester): Added.
(BuildbotSyncer.prototype.matchesConfiguration):
(BuildbotSyncer.prototype._propertiesForBuildRequest): Updated to support the new format.
(BuildbotSyncer._loadConfig): Ditto. Optionally parse buildConfigurations.
(BuildbotSyncer._resolveBuildersWithPlatforms): Added. For each test or build configuration entry,
creates the list of configurations per builder and platform.
(BuildbotSyncer._parseRepositoryGroup): Added the support for parsing the new format with revision,
roots, and patch option types with a lot of validations as we're seeing a bit of combinatorial
explosion in the number of things that can go wrong. Also parse buildProperties optionally.
(BuildbotSyncer._parseRepositoryGroupPropertyTemplate): Added. A helper function to parse a set of
buildbot properties, validates its content, and invokes a callback if it's an dynamically resolved
type such as "revision" and "patch".
(BuildbotSyncer._validateAndMergeConfig): Updated per the format change. No longer allows "types",
"type", "platforms", and "platform" as they're explicity resolved in _resolveBuildersWithPlatforms.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.syncOnce):
(BuildbotTriggerable.prototype._validateRequests): Handle the case when a build request is not
associated with any test.
(BuildbotTriggerable.prototype._nextRequestInGroup): Return null when there is a build request to
build a patch which has not been completed (pending, scheduled, running, or failed). Since all
requests to build a patch has a negative order, those requests should all show up at the beginning.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Pick a new buildbot syncer when
scheduling the first request to build a patch or the first request to run a test. The first request
to run a test will always have order of 0, so it's a sufficient condition to find such a request.
On the other hand, the first request to build a patch can have a negative order number so we must
explicitly check if it's the first item in the ordered list of requests in the test group.
* tools/remote-server-relay.log: Added.
* tools/sync-buildbot.js:
(syncLoop): Fixed a bug we were not re-fetching the triggerable after updating the triggerable so
that Triggerable and related objects we have in the memory may not reflect what we just synced to
the perf dashboard. Also, we don't reset V3 models manually any more since Manifest.fetch does that.
* unit-tests/buildbot-syncer-tests.js: Added more test cases and updated existing test cases to test
exception messages explicitly since allowing any exception was resulting in some tests passing a
result of unrelated parsing error being thrown, etc...
(sampleiOSConfig): Updated per the format change.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto.
Canonical link: https://commits.webkit.org/189454@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@217378 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-05-24 18:47:01 +00:00
|
|
|
}).then(() => Manifest.fetch()).then(() => {
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
const repositoryGroups = TriggerableRepositoryGroup.sortByName(TriggerableRepositoryGroup.all());
|
|
|
|
const webkit = Repository.findTopLevelByName('WebKit');
|
|
|
|
const macos = Repository.findTopLevelByName('macOS');
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 2);
|
|
|
|
assert.strictEqual(repositoryGroups[0].name(), 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[0].description(), 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[0].acceptsCustomRoots(), false);
|
|
|
|
assert.deepStrictEqual(repositoryGroups[0].repositories(), [webkit, macos]);
|
|
|
|
assert.strictEqual(repositoryGroups[0].acceptsPatchForRepository(webkit), true);
|
|
|
|
assert.strictEqual(repositoryGroups[0].acceptsPatchForRepository(macos), false);
|
|
|
|
|
|
|
|
assert.strictEqual(repositoryGroups[1].name(), 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[1].description(), 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[1].acceptsCustomRoots(), true);
|
|
|
|
assert.deepStrictEqual(repositoryGroups[1].repositories(), [macos]);
|
|
|
|
assert.strictEqual(repositoryGroups[1].acceptsPatchForRepository(webkit), false);
|
|
|
|
assert.strictEqual(repositoryGroups[1].acceptsPatchForRepository(macos), false);
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', initialUpdate);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
it('should replace a repository when the repository group name matches', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
const initialUpdate = updateWithMacWebKitRepositoryGroups();
|
|
|
|
const secondUpdate = updateWithMacWebKitRepositoryGroups();
|
|
|
|
let initialGroups;
|
Add the support for scheduling a A/B testing with a patch.
https://bugs.webkit.org/show_bug.cgi?id=171209
Reviewed by Chris Dumez.
Added the support for creating a custom test group with a patch applied.
First, each repository in a repository group has a boolean indicating whether a given repository can have
a patch applied or not. When any configuration in a test group contains a patch, we create build requests
without a test specified in order to "build" those patches. These build requests have negative order numbers
to differentiate them from regular build requests. We can't simply build ones with patches since there could
be differences in SDK, build options, etc... when patches are applied.
The JSON format for commit sets returned by /api/build-requests have been changed from using an array of
commit IDs to an array of dictionaries indicate commit and acceptsPatch boolean. /api/update-triggerable now
uses a dictionary with two keys: repository and acceptsPatch to specify a set of repositories associated with
a repository group, and /privileged-api-create-test-group uses a dictionary with two keys: revision and patch
instead of a revision string to specify commit sets.
Furthermore, the syncing script's configuration have been updated to use a dictionary of repository names to
an options dictionary instead of an array of repositories names. For now, the only supported option is
acceptsPatch but will be extended when we add the support for rolling back system components.
e.g. {"WebKit": {acceptsPatch: true}, "macOS": {}} instead of ["WebKit", "macOS"]
On the UI side, InstantFileUploader has been changed to accept only one file by default, and added a new method
allowMultipleFiles() to allow multiple files to be selected for custom roots. Also replaced the input element
with type=file by a button with a custom label to show labels such as "Apply a patch" or "Add a new root"
instead of the generic label like "choose a file".
* init-database.sql: Added trigrepo_accepts_patch to triggerable_repositories to indicate whether a given
repository can have a patch applied or not. Made request_test optional in build_requests for when a build
request is created to build patches. Such a build request have a negative request_order. Updated the related
constraints accordingly.
* public/admin/triggerables.php: Added the support for updating whether a given repository can have a patch
applied in each repository group. Only show the repositories in the repository group for this purpose since
there is no way to accept a patch on a repository without it being a part of the group.
(generate_repository_form): Now takes the markup for checkboxes instead of generating one itself.
(generate_repository_checkboxes): Now takes an array of repositories to generate checkboxes. The checkbox is
shown when the repository ID exists as a key in this array, and is checked when its value is true. The new
capability to skip repositories not in the array is used to hide repositories not associated with the group
in the list of checkboxes to indicate a repository accepts a patch.
* public/api/update-triggerable.php:
(main): Now updates the description and acceptsRoots states of each repository group, and sets acceptsPatch
boolean for each repository in the group if set in the update.
(validate_repository_groups): Use a reference to $repository_groups in order to set repository_id_list, which
contains an array of repository IDs to find the existing repository group that matches the set via
RepositoryGroupFinder's find_by_repositories. Also added a various validations for acceptsRoots, a dictionary
specifying repository and acceptsPatch.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::fetch_commits_for_set_if_needed): Instead of returning an array of commit IDs as
"commits", it now returns an array of dictionaries with "commit" and "patch" keys specifying the commit ID
and the patch file's ID respectively as "revisionItems".
(BuildRequestsFetcher::add_uploaded_file): Added. Extracted from fetch_commits_for_set_if_needed. Used to
add either a patch file or a custom root file in the list of uploaded files in the result.
* public/include/manifest-generator.php:
(fetch_triggerables): Each element in repository group's "repositories" field is now an array of dictionaries
with "repository" and "acceptsPatch" as keys.
* public/include/repository-group-finder.php:
(RepositoryGroupFinder::__construct): Added a map for boolean indicating whether a given repository group
allows a patch on a repository. Used in /privileged-api/create-test-group.
(RepositoryGroupFinder::accepts_patch): Added.
(RepositoryGroupFinder::populate_map): Build up the map for acceptsPatch boolean per repository per group.
* public/privileged-api/create-test-group.php:
(main): Fixed a bug that we were not explicitly checking for a duplicate test group name (with a test). Create
build requests to "build" patches if there is any patch file specified.
(commit_sets_from_revision_sets): Updated to take a dictionary with "revision" and "patch" as keys to specify
a revision and a patch if any instead of just a revision string for each repository. Also validate that each
repository is allowed to have a patch once the repository group has been found for the set of repositories.
(ensure_commit_sets):
* public/v3/components/custom-analysis-task-configurator.js:
(CustomAnalysisTaskConfigurator): Added _patchUploaders as an instance variable, which is a dictionary of
configuration names to a map of InstantFileUploader's used to upload a patch. Also renamed _fileUploaders to
_customRootUploaders for clarity.
(CustomAnalysisTaskConfigurator.prototype.setCommitSets):
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree.createRootUploader): Added.
(CustomAnalysisTaskConfigurator.prototype.didConstructShadowTree):
(CustomAnalysisTaskConfigurator.prototype._ensurePatchUploader): Added. Creates an instant file uploader for
patches. We only allow a single patch per repository.
(CustomAnalysisTaskConfigurator.prototype._computeCommitSet): Include a patch in the commit set as needed.
(CustomAnalysisTaskConfigurator.prototype._buildRevisionTable): Show the patch file uploader for repositories
which can have patches in the current repository group.
(CustomAnalysisTaskConfigurator.cssTemplate): Show borders between every rows instead of just between tbody's
now that each row can have a patch file uploader.
* public/v3/components/instant-file-uploader.js:
(InstantFileUploader): Added _fileInput and _allowMultipleFiles as instance variables. We now show a button
in the UI instead of an input with type=file. _fileInput is a hidden input with type=file used inside a click
event of the button to let the user pick a file.
(InstantFileUploader.prototype.allowMultipleFiles): Added. Allows this instance to accept multiple files.
(InstantFileUploader.prototype.didConstructShadowTree): Synthetically click on the hidden input element when
the newly added button element is clicked to open the browser's file picker.
(InstantFileUploader.prototype.render): Hide the button to add a file if this instance can only select one file
and there is already some file being uploaded in this instance.
(InstantFileUploader.htmlTemplate): Replaced the input element with type=file with a button. Its label comes
from the default slot content.
* public/v3/models/build-request.js:
(BuildRequest): Made the test optional.
(BuildRequest.prototype.isBuild): Returns true if this is a build request for building a patch.
(BuildRequest.prototype.isTest): Returns true if this is a build request for running tests.
(BuildRequest.constructBuildRequestsFromData): Create each commit log here instead of relying on CommitSet's
constructor to construct its commit logs. Also updated per the replacement of an array of commit IDs by
an array of dictionaries with commit and patch properties.
* public/v3/models/commit-set.js:
(CommitSet): Made _repositoryToCommitMap a real Map object. Also added _repositoryToPatchMap. Also got rid of
the code to instantiate commit logs since that's now done in BuildRequest.constructBuildRequestsFromData.
(CommitSet.prototype.commitForRepository):
(CommitSet.prototype.revisionForRepository):
(CommitSet.prototype.patchForRepository): Added.
(CommitSet.prototype.latestCommitTime): Modernized the code.
(CommitSet.prototype.equals): Modernized the code. Also added the check for patches.
(MeasurementCommitSet): Updated per the change to make _repositoryToCommitMap a real Map.
(CustomCommitSet.prototype.setRevisionForRepository):
(CustomCommitSet.prototype.equals): Added the check for patches.
(CustomCommitSet.prototype.revisionForRepository):
(CustomCommitSet.prototype.patchForRepository): Added.
* public/v3/models/manifest.js:
(Manifest._didFetchManifest): Updated per the replacement of an array of commit IDs by an array of dictionaries
with commit and patch properties.
* public/v3/models/repository.js:
(Repository.prototype.ownerId): Renamed from owner for clarity.
* public/v3/models/test-group.js:
(TestGroup): Modernized the code by using LazilyEvaluatedFunction. Removed _requestsAreInOrder since it's not
necessary anymore with LazilyEvaluatedFunction.
(TestGroup.prototype.addBuildRequest):
(TestGroup.prototype.test): Use the last build request's test since the first few requests could be requests to
build patches.
(TestGroup.prototype.platform): Ditto.
(TestGroup.prototype._lastRequest): Added.
(TestGroup.prototype._orderedBuildRequests): Added.
(TestGroup.prototype.repetitionCount): Only count the build requests for testing (skipping any requests to
build patches).
(TestGroup.prototype.requestedCommitSets): Simply call _computeRequestedCommitSetsLazily.
(TestGroup.prototype._computeRequestedCommitSets): Extracted from requestedCommitSets.
(TestGroup.prototype.requestsForCommitSet):
(TestGroup.prototype.labelForCommitSet): Rewritten. Just compute the label here instead of relying on
_commitSetToLabel since requestedSets is always of the length two at the moment.
(TestGroup._revisionSetsFromCommitSets): Specify both the revision and the patch in the revision set.
* public/v3/models/triggerable.js:
(TriggerableRepositoryGroup): Added _patchAcceptingSet as an instance variable. Use
sortByNamePreferringOnesWithURL to sort repositories instead of simple sortByName.
(TriggerableRepositoryGroup.prototype.accepts): Added checks for the custom roots and patches.
(TriggerableRepositoryGroup.prototype.acceptsPatchForRepository): Added.
* server-tests/api-build-requests-tests.js: Updated the test cases per the replacement of an array of commit
IDs by an array of dictionaries with commit and patch properties.
* server-tests/api-manifest-tests.js: Updated the test case per the name of Repository's owner to ownerId.
* server-tests/api-update-triggerable.js: Updated the test case per the name of Repository's owner to ownerId,
and added a test case for updating whether a given repository group allows custom roots as well as patches
on repositories via /api/update-triggerable.
(.updateWithOSXRepositoryGroup): Updated the sample syncing script configuration per the format change.
(.refetchManifest): Added.
* server-tests/privileged-api-create-test-group-tests.js: Updated per the syncing script configuration format
change. Also added a test for creating a test group with a duplicate name, which is expected to fail with
DuplicateTestGroupName, and creating a test group with a patch both when it's allowed and when it's not allowed
in the matching repository group.
(.addTriggerableAndCreateTask): Updated per the format change.
* server-tests/resources/mock-data.js:
(MockData.addEmptyTriggerable): Added a metric and its configuration to make it appear in the manifest file.
The new test case in api-update-triggerable.js requires this.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated per the syncing script configuration format change.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Removed the useless assertions about test configurations,
and added assertions about custom roots and patches in the test case for updateTriggerables.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer._parseRepositoryGroup): Made each assertion explicitly refer to the specific repository group
to make it more user friendly. Now each repository group uses a dictionary of repository names to its options
in the syncing script configurations. When parsed, we insert it as an array of dictionaries with repository ID
and acceptsPatch boolean specified separately since this is the format /api/update-triggerable expects.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable):
* unit-tests/build-request-tests.js:
(sampleBuildRequestData): Updated per the commit sets format change in /api/build-requests.
* unit-tests/buildbot-syncer-tests.js: Updated the existing tests per various format changes and added a couple
of new test cases for the syncing script's configuration validation.
(sampleiOSConfig):
(smallConfiguration):
(createSampleBuildRequest):
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Updated per the repository group format change.
* unit-tests/test-groups-tests.js:
(sampleTestGroup): Updated per the commit sets format change in /api/build-requests.
Canonical link: https://commits.webkit.org/188378@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215987 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-30 18:11:47 +00:00
|
|
|
secondUpdate.repositoryGroups[1].repositories[0] = {repository: MockData.gitWebkitRepositoryId()}
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(initialUpdate);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', initialUpdate);
|
|
|
|
}).then((response) => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_repository_groups', 'name'), db.selectAll('triggerable_repositories', 'repository')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [repositoryGroups, repositories] = result;
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 2);
|
|
|
|
assert.strictEqual(repositoryGroups[0]['name'], 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[0]['triggerable'], MockData.emptyTriggeragbleId());
|
|
|
|
assert.strictEqual(repositoryGroups[1]['name'], 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[1]['triggerable'], MockData.emptyTriggeragbleId());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
initialGroups = repositoryGroups;
|
|
|
|
|
|
|
|
const repositoriesByGroup = mapRepositoriesByGroup(repositories);
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(Object.keys(repositoriesByGroup).length, 2);
|
|
|
|
assert.deepStrictEqual(repositoriesByGroup[repositoryGroups[0]['id']], [MockData.macosRepositoryId(), MockData.webkitRepositoryId()]);
|
|
|
|
assert.deepStrictEqual(repositoriesByGroup[repositoryGroups[1]['id']], [MockData.macosRepositoryId()]);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', secondUpdate);
|
|
|
|
}).then(() => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_repository_groups', 'name'), db.selectAll('triggerable_repositories', 'repository')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [repositoryGroups, repositories] = result;
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.deepStrictEqual(repositoryGroups, initialGroups);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
|
|
|
|
const repositoriesByGroup = mapRepositoriesByGroup(repositories);
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(Object.keys(repositoriesByGroup).length, 2);
|
|
|
|
assert.deepStrictEqual(repositoriesByGroup[initialGroups[0]['id']], [MockData.macosRepositoryId(), MockData.gitWebkitRepositoryId()]);
|
|
|
|
assert.deepStrictEqual(repositoriesByGroup[initialGroups[1]['id']], [MockData.macosRepositoryId()]);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
it('should replace a repository when the list of repositories matches', () => {
|
|
|
|
const db = TestServer.database();
|
|
|
|
const initialUpdate = updateWithMacWebKitRepositoryGroups();
|
|
|
|
const secondUpdate = updateWithMacWebKitRepositoryGroups();
|
|
|
|
let initialGroups;
|
|
|
|
let initialRepositories;
|
|
|
|
secondUpdate.repositoryGroups[0].name = 'mac-only';
|
|
|
|
return MockData.addEmptyTriggerable(db).then(() => {
|
2021-03-22 21:23:38 +00:00
|
|
|
return addWorkerForReport(initialUpdate);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
}).then(() => {
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', initialUpdate);
|
|
|
|
}).then((response) => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_repository_groups', 'name'), db.selectAll('triggerable_repositories', 'repository')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [repositoryGroups, repositories] = result;
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 2);
|
|
|
|
assert.strictEqual(repositoryGroups[0]['name'], 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[0]['triggerable'], MockData.emptyTriggeragbleId());
|
|
|
|
assert.strictEqual(repositoryGroups[1]['name'], 'system-only');
|
|
|
|
assert.strictEqual(repositoryGroups[1]['triggerable'], MockData.emptyTriggeragbleId());
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
initialGroups = repositoryGroups;
|
|
|
|
|
|
|
|
const repositoriesByGroup = mapRepositoriesByGroup(repositories);
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(Object.keys(repositoriesByGroup).length, 2);
|
|
|
|
assert.deepStrictEqual(repositoriesByGroup[repositoryGroups[0]['id']], [MockData.macosRepositoryId(), MockData.webkitRepositoryId()]);
|
|
|
|
assert.deepStrictEqual(repositoriesByGroup[repositoryGroups[1]['id']], [MockData.macosRepositoryId()]);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
initialRepositories = repositories;
|
|
|
|
|
|
|
|
return TestServer.remoteAPI().postJSONWithStatus('/api/update-triggerable/', secondUpdate);
|
|
|
|
}).then(() => {
|
|
|
|
return Promise.all([db.selectAll('triggerable_repository_groups', 'name'), db.selectAll('triggerable_repositories', 'repository')]);
|
|
|
|
}).then((result) => {
|
|
|
|
const [repositoryGroups, repositories] = result;
|
|
|
|
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.strictEqual(repositoryGroups.length, 2);
|
|
|
|
assert.strictEqual(repositoryGroups[0]['name'], 'mac-only');
|
|
|
|
assert.strictEqual(repositoryGroups[0]['triggerable'], initialGroups[1]['triggerable']);
|
|
|
|
assert.strictEqual(repositoryGroups[1]['name'], 'system-and-webkit');
|
|
|
|
assert.strictEqual(repositoryGroups[1]['triggerable'], initialGroups[0]['triggerable']);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
|
2021-03-06 14:31:56 +00:00
|
|
|
assert.deepStrictEqual(repositories, initialRepositories);
|
Each build request should be associated with a repository group
https://bugs.webkit.org/show_bug.cgi?id=170528
Rubber-stamped by Chris Dumez.
Make the buildbot syncing script use the concept of repository groups so that each repository group can post
a different set of properties to buildbot. In order to do this, we associate each build request with
a repository group to use. Each triggerable's repository groups is now updated by the syncing scripts via
/api/update-triggerable just the same way the set of the supported platform, test pairs are updated.
Each repository group specifies the list of repositories, a dictionary that maps the buildbot property name
to either a string value or a repository name enclosed in < and >:
```js
"repositoryGroups": {
"webkit-svn": {
"repositories": ["WebKit", "macOS"],
"properties": {"os": "<macOS>", "wk": "<WebKit>"}
}
}
```
With this, removed the support for specifying a repository to use in generic dictionary of properties via
a dictionary with a single key of "root", "rootOptions", and "rootsExcluding". We now validate that the list of
repositories in each repository group matches exactly the ones used in buildbot properties as well as ones in
build requests.
After this patch, sync-with-buildbot.js will no longer schedule a build request without a repository group.
Run the appropriate database queries to set the repository group on each build request. Because of this change,
this patch also makes BuildbotTriggerable.prototype.syncOnce more robust against invalid build requests.
Instead of throwing an exception and exiting early, it simply skips all build requests that belong to the same
test group if the next build request to be scheduled does not specify a repository group.
* init-database.sql: Add request_repository_group column to build_requests table, and a unique constraint for
repository and group pair in triggerable_repositories table.
* public/api/update-triggerable.php:
(main): Validate and insert repository groups.
(validate_configurations): Extracted from main.
(validate_repository_groups): Added.
* public/v3/models/repository.js:
(Repository.findTopLevelByName): Added.
* public/include/build-requests-fetcher.php:
(BuildRequestsFetcher::results_internal): Include the repository group of each request in the JSON response.
* public/include/repository-group-finder.php: Added. A helper class to find the repository group for a given
triggerable for a list of repositories.
(RepositoryGroupFinder): Added.
(RepositoryGroupFinder::__construct): Added.
(RepositoryGroupFinder::find_by_repositories): Added.
(RepositoryGroupFinder::populate_map): Added.
* public/privileged-api/create-test-group.php:
(main): Each element in an array returned by ensure_commit_sets and commit_sets_from_revision_sets now contains
"set", the list of commit IDs, and "repository_group", the repository group identified for each commit set.
Use that to set the repository group in each new build request.
(commit_sets_from_revision_sets): Use RepositoryGroupFinder to find the right repository group.
(ensure_commit_sets): Ditto. There is no need to find a repository group for each commit set here since its
argument is keyed by the repository name. e.g. {"WebKit": [123, 456], "macOS": ["16A323", "16A323"]}
* public/v3/models/build-request.js:
(BuildRequest):
(BuildRequest.prototype.triggerable): Added.
(BuildRequest.prototype.repositoryGroup): Added.
(BuildRequest.constructBuildRequestsFromData): Resolve the triggerable and the repository group.
* public/v3/models/triggerable.js:
(Triggerable.prototype.name): Added.
(Triggerable.prototype.acceptedRepositories): Deleted.
(TriggerableRepositoryGroup):
(TriggerableRepositoryGroup.prototype.accepts): Added. Retruns true if the repository group
* server-tests/api-build-requests-tests.js: Added a test for getting the repository group of a build request.
* server-tests/api-manifest-tests.js: Added assertions for the repository groups.
* server-tests/api-report-tests.js:
(.emptyReport):
(.reportWithTwoLevelsOfAggregations):
* server-tests/api-update-triggerable.js: Added test cases for updating the repository groups associated with
a triggerable.
(.updateWithOSXRepositoryGroup):
(.mapRepositoriesByGroup):
* server-tests/privileged-api-create-test-group-tests.js:
(addTriggerableAndCreateTask): Add two repository groups for testing. Added assertions for repository groups
in existing test cases, and added a test case for creating a test group with two different repository groups.
* server-tests/resources/mock-data.js:
(MockData.resetV3Models): Reset TriggerableRepositoryGroup's static maps.
(MockData.emptyTriggeragbleId): Added.
(MockData.macosRepositoryId): Added.
(MockData.webkitRepositoryId): Added.
(MockData.gitWebkitRepositoryId): Added.
(MockData.addMockData): Create repository groups as needed. Renamed the "OS X" repository to "macOS" since some
tests were using the latter, and now we need mock data to be consistent across tests due to stricter checks.
(MockData.addEmptyTriggerable): Added. Used in api-update-triggerable.js.
(MockData.addMockTestGroupWithGitWebKit): Added. Used in api-build-requests-tests.js.
(MockData.addAnotherMockTestGroup): Cleanup.
(MockData.mockTestSyncConfigWithSingleBuilder): Updated the mock configuration per code changes.
(MockData.mockTestSyncConfigWithTwoBuilders): Ditto.
* server-tests/tools-buildbot-triggerable-tests.js: Updated a test case testing /api/update-triggerable to test
updating the set of repository groups in addition to the set of test, platform pairs.
(.refetchManifest): Added.
* tools/js/buildbot-syncer.js:
(BuildbotSyncer): Now takes a set of configurations shared across syncers: repositoryGroups, slaveArgument,
and buildRequestArgument as the third argument.
(BuildbotSyncer.prototype.repositoryGroups): Added.
(BuildbotSyncer.prototype._testGroupMapForBuildRequests): Cleaned up the code to use Array.prototype.find.
Also added an assertion that the build request is associated with a repository group.
(BuildbotSyncer.prototype._propertiesForBuildRequest): Removed the support for using an arbitary property to
specify a revision in favor of explicity listing each property and repository name in a repository group.
(BuildbotSyncer._loadConfig): Removed the support for "shared", which specified the set of buildbot properties
shared across syncers, the name of properties which specifies the build slave name and build request ID. These
values are not stored as top-level properties and superseded by the concept of repository groups.
(BuildbotSyncer._parseRepositoryGroup): Parses and validates repository groups.
(BuildbotSyncer._createTestConfiguration): We no longer expect each configuration to specify a dictionary of
properties or buildRequestArgument (often inherited from shared).
(BuildbotSyncer._validateAndMergeConfig): Removed "slaveArgument" and "buildRequestArgument" from the list of
allowed proeprties in each configuration now that they're specified as top-level properties.
* tools/js/buildbot-triggerable.js:
(BuildbotTriggerable.prototype.updateTriggerable): Update the associated repository groups.
(BuildbotTriggerable.prototype.syncOnce): Skip test groups for which the next build request to be scheduled is
not included in the list of valid build requests.
(BuildbotTriggerable.prototype._validateRequests): Now returns the list of valid build requests, which excludes
those that lack a repository group set.
(BuildbotTriggerable.prototype._nextRequestInGroup): Extracted from _scheduleRequestIfSlaveIsAvailable. Finds
the next build request to be scheduled for the test group.
(BuildbotTriggerable.prototype._scheduleRequestIfSlaveIsAvailable): Renamed from
_scheduleNextRequestInGroupIfSlaveIsAvailable. Now takes the syncer and the slave name as arguments instead of
a test group information since syncOnce now calls _nextRequestInGroup to find the next build request.
* tools/js/v3-models.js:
* unit-tests/build-request-tests.js: Fixed the test name.
* unit-tests/buildbot-syncer-tests.js: Removed tests for "rootOptions" and "rootsExcluding", and added tests
for parsing repository groups.
(sampleiOSConfig): Updated the mock configuration per code changes.
(sampleiOSConfigWithExpansions): Ditto.
(smallConfiguration): Ditto. Now returns the entire configuration instead of a single builder configuration.
Various test cases have been updated to reflect this.
(createSampleBuildRequest): Removed the git hash of WebKit to match the repository groups listed in the mock
configurations. The git hash was there to test "rootOptions", which this patch removed.
(samplePendingBuild): Removed "root_dict" from the list of properties. This was used to test "rootsExcluding"
which, again, this patch removed.
(sampleInProgressBuild): Ditto.
(sampleFinishedBuild): Ditto.
* unit-tests/resources/mock-v3-models.js:
(MockModels.inject): Added ock repository groups so that existing tests will continue to function.
Canonical link: https://commits.webkit.org/187490@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215061 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2017-04-06 21:56:59 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
2016-06-08 20:43:01 +00:00
|
|
|
});
|