229 lines
11 KiB
Python
229 lines
11 KiB
Python
# Copyright (c) 2014, Canon Inc. All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
# 3. Neither the name of Canon Inc. nor the names of
|
|
# its contributors may be used to endorse or promote products derived
|
|
# from this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY
|
|
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR
|
|
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
"""
|
|
This script downloads W3C test repositories.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
|
|
from webkitpy.common.system.filesystem import FileSystem
|
|
from webkitpy.common.webkit_finder import WebKitFinder
|
|
from webkitpy.common.checkout.scm.git import Git
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
|
|
class TestDownloader(object):
|
|
|
|
@staticmethod
|
|
def default_options():
|
|
options = type('', (), {})
|
|
options.fetch = True
|
|
options.verbose = False
|
|
options.import_all = False
|
|
return options
|
|
|
|
@staticmethod
|
|
def load_test_repositories(filesystem=FileSystem()):
|
|
webkit_finder = WebKitFinder(filesystem)
|
|
test_repositories_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'TestRepositories')
|
|
return json.loads(filesystem.read_text_file(test_repositories_path))
|
|
|
|
def __init__(self, repository_directory, host, options):
|
|
self._options = options
|
|
self._host = host
|
|
self._filesystem = host.filesystem
|
|
self._test_suites = []
|
|
|
|
self.repository_directory = repository_directory
|
|
|
|
self.test_repositories = self.load_test_repositories(self._filesystem)
|
|
|
|
self.paths_to_skip = []
|
|
self.paths_to_import = []
|
|
for test_repository in self.test_repositories:
|
|
self.paths_to_skip.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_skip']])
|
|
self.paths_to_import.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_import']])
|
|
|
|
webkit_finder = WebKitFinder(self._filesystem)
|
|
self.import_expectations_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'import-expectations.json')
|
|
if not self._filesystem.isfile(self.import_expectations_path):
|
|
_log.warning('Unable to read import expectation file: %s' % self.import_expectations_path)
|
|
if not self._options.import_all:
|
|
self._init_paths_from_expectations()
|
|
|
|
def git(self, test_repository):
|
|
return Git(test_repository, None, executive=self._host.executive, filesystem=self._filesystem)
|
|
|
|
def checkout_test_repository(self, revision, url, directory):
|
|
git = None
|
|
if not self._filesystem.exists(directory):
|
|
_log.info('Cloning %s into %s...' % (url, directory))
|
|
Git.clone(url, directory, self._host.executive)
|
|
git = self.git(directory)
|
|
elif self._options.fetch is True:
|
|
git = self.git(directory)
|
|
_log.info('Fetching %s...' % url)
|
|
git.fetch()
|
|
else:
|
|
git = self.git(directory)
|
|
_log.info('Checking out revision ' + revision)
|
|
git.checkout(revision, not self._options.verbose)
|
|
|
|
def _init_paths_from_expectations(self):
|
|
import_lines = json.loads(self._filesystem.read_text_file(self.import_expectations_path))
|
|
for path, policy in import_lines.items():
|
|
if policy == 'skip':
|
|
self.paths_to_skip.append(path)
|
|
elif policy == 'import':
|
|
self.paths_to_import.append(path)
|
|
else:
|
|
_log.warning('Problem reading import lines ' + path)
|
|
|
|
def update_import_expectations(self, test_paths):
|
|
import_lines = json.loads(self._filesystem.read_text_file(self.import_expectations_path))
|
|
for path in test_paths:
|
|
import_lines[path] = 'import'
|
|
self._filesystem.write_text_file(self.import_expectations_path, json.dumps(import_lines, sort_keys=True, indent=4))
|
|
|
|
def _add_test_suite_paths(self, test_paths, directory, webkit_path):
|
|
for name in self._filesystem.listdir(directory):
|
|
original_path = self._filesystem.join(webkit_path, name)
|
|
if not name.startswith('.') and not original_path in self.paths_to_skip:
|
|
test_paths.append(original_path)
|
|
|
|
def _empty_directory(self, directory):
|
|
if self._filesystem.exists(directory):
|
|
self._filesystem.rmtree(directory)
|
|
self._filesystem.maybe_make_directory(directory)
|
|
|
|
def copy_tests(self, destination_directory, test_paths):
|
|
for test_repository in self.test_repositories:
|
|
self._empty_directory(self._filesystem.join(destination_directory, test_repository['name']))
|
|
|
|
copy_paths = []
|
|
if test_paths:
|
|
for path in test_paths:
|
|
copy_paths.append(path)
|
|
for path in self.paths_to_import:
|
|
copy_paths.append(path)
|
|
else:
|
|
for test_repository in self.test_repositories:
|
|
self._add_test_suite_paths(copy_paths, self._filesystem.join(self.repository_directory, test_repository['name']), test_repository['name'])
|
|
# Handling of tests marked as [ Pass ] in expectations file.
|
|
for path in self.paths_to_import:
|
|
if not path in copy_paths:
|
|
copy_paths.append(path)
|
|
|
|
def longest_path(filesystem, paths):
|
|
longest_matching_path = ""
|
|
for path in paths:
|
|
if path.startswith(longest_matching_path):
|
|
longest_matching_path = path
|
|
return longest_matching_path
|
|
|
|
def should_copy_dir(filesystem, directory):
|
|
relative_path = self._filesystem.relpath(directory, self.repository_directory)
|
|
if relative_path == ".":
|
|
return True
|
|
|
|
potential_copy_paths = [copy_directory for copy_directory in copy_paths if relative_path.startswith(copy_directory)]
|
|
if (not potential_copy_paths):
|
|
return False
|
|
|
|
potential_skip_paths = [skip_directory for skip_directory in self.paths_to_skip if relative_path.startswith(skip_directory)]
|
|
if (not potential_skip_paths):
|
|
return True
|
|
|
|
longest_copy_path = longest_path(filesystem, potential_copy_paths)
|
|
longest_skip_path = longest_path(filesystem, potential_skip_paths)
|
|
return longest_copy_path.startswith(longest_skip_path)
|
|
|
|
# Compute directories for which we should copy direct children
|
|
directories_to_copy = self._filesystem.dirs_under(self.repository_directory, should_copy_dir)
|
|
|
|
def should_copy_file(filesystem, dirname, filename):
|
|
full_path = filesystem.join(dirname, filename)
|
|
relative_path = self._filesystem.relpath(full_path, self.repository_directory)
|
|
if relative_path in copy_paths:
|
|
return True
|
|
if relative_path in self.paths_to_skip:
|
|
return False
|
|
return dirname in directories_to_copy
|
|
|
|
for source_path in self._filesystem.files_under(self.repository_directory, file_filter=should_copy_file):
|
|
destination_path = self._filesystem.join(destination_directory, self._filesystem.relpath(source_path, self.repository_directory))
|
|
self._filesystem.maybe_make_directory(self._filesystem.dirname(destination_path))
|
|
self._filesystem.copyfile(source_path, destination_path)
|
|
|
|
def _git_submodules_description(self, test_repository):
|
|
directory = self._filesystem.join(self.repository_directory, test_repository['name'])
|
|
|
|
git = self.git(directory)
|
|
git.init_submodules()
|
|
|
|
submodules = []
|
|
submodules_status = [line.strip().split(' ') for line in git.submodules_status().splitlines()]
|
|
for status in submodules_status:
|
|
version = status[0]
|
|
path = status[1].split('/')
|
|
|
|
url = self.git(self._filesystem.join(directory, status[1])).origin_url()
|
|
if not url.startswith('https://github.com/'):
|
|
_log.warning('Submodule %s (%s) is not hosted on github' % (status[1], url))
|
|
_log.warning('Please ensure that generated URL points to an archive of the module or manually edit its value after the import')
|
|
url = url[:-4] # to remove .git
|
|
|
|
submodule = {}
|
|
submodule['path'] = path
|
|
submodule['url'] = url + '/archive/' + version + '.tar.gz'
|
|
submodule['url_subpath'] = url.split('/').pop() + '-' + version
|
|
submodules.append(submodule)
|
|
|
|
git.deinit_submodules()
|
|
return submodules
|
|
|
|
def generate_git_submodules_description(self, test_repository, filepath):
|
|
self._filesystem.write_text_file(filepath, json.dumps(self._git_submodules_description(test_repository), sort_keys=True, indent=4))
|
|
|
|
def generate_gitignore(self, test_repository, destination_directory):
|
|
rules = []
|
|
for submodule in self._git_submodules_description(test_repository):
|
|
path = list(submodule['path'])
|
|
path.insert(0, '')
|
|
rules.append('/'.join(path[:-1]) + '/' + path[-1] + '/')
|
|
rules.append('/'.join(path[:-1]) + '/.' + path[-1] + '.url')
|
|
self._filesystem.write_text_file(self._filesystem.join(destination_directory, test_repository['name'], '.gitignore'), '\n'.join(rules))
|
|
|
|
def clone_tests(self, use_tip_of_tree=False):
|
|
for test_repository in self.test_repositories:
|
|
self.checkout_test_repository(test_repository['revision'] if not use_tip_of_tree else 'origin/master', test_repository['url'], self._filesystem.join(self.repository_directory, test_repository['name']))
|
|
|
|
def download_tests(self, destination_directory, test_paths=[], use_tip_of_tree=False):
|
|
self.clone_tests(use_tip_of_tree)
|
|
self.copy_tests(destination_directory, test_paths)
|