480 lines
17 KiB
Ruby
480 lines
17 KiB
Ruby
# Copyright (C) 2017 Sony Interactive Entertainment Inc.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
#
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
|
|
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
|
|
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
require 'open3'
|
|
|
|
$hasDiff = false
|
|
if ($hostOS == 'windows')
|
|
out, err, status = Open3.capture3("where", "/q", "diff")
|
|
$hasDiff = status.success?
|
|
else
|
|
out, err, status = Open3.capture3("which", "diff")
|
|
$hasDiff = status.success?
|
|
end
|
|
|
|
# Prefix each line of str with the name
|
|
def prefixString(str, name)
|
|
"#{str}.empty? ? \"\" : #{str}.gsub(/^/m, \"#{name}: \")"
|
|
end
|
|
|
|
def silentOutputHandler
|
|
Proc.new {
|
|
| name |
|
|
<<-END_SILENT_OUTPUT_HANDLER
|
|
out = out + err
|
|
err = nil
|
|
STDOUT.puts #{prefixString("out", name)} if (!out.empty?)
|
|
File.open("#{Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)}", "w") do |out_file|
|
|
out_file.puts out
|
|
end
|
|
END_SILENT_OUTPUT_HANDLER
|
|
}
|
|
end
|
|
|
|
# Output handler for tests that are expected to produce meaningful output.
|
|
def noisyOutputHandler
|
|
Proc.new {
|
|
| name |
|
|
<<-END_NOISY_OUTPUT_HANDLER
|
|
out = out + err
|
|
err = nil
|
|
File.open("#{Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)}", "w") do |out_file|
|
|
out_file.puts out
|
|
end
|
|
END_NOISY_OUTPUT_HANDLER
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that fail exactly when they return non-zero exit status.
|
|
# This is useful when a test is expected to fail.
|
|
def simpleErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if !success(status)\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code \#{status.exitstatus}\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "else\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that fail exactly when they return zero exit status.
|
|
def expectedFailErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if success(status)\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code 0\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "else\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that fail exactly when they return non-zero exit status and produce
|
|
# lots of spew. This will echo that spew when the test fails.
|
|
def noisyErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if !success(status)\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code \#{status.exitstatus}\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "else\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
def fallbackDiff(expected, output)
|
|
<<-END_FALLBACK_DIFF
|
|
# Fallback diff for when diff(1) isn't available
|
|
diffs = []
|
|
line_number = 0
|
|
File.open("#{expected}") do | expected |
|
|
File.open("#{output}") do | actual |
|
|
loop do
|
|
l1 = expected.gets
|
|
l2 = actual.gets
|
|
if (l1 != l2)
|
|
diffs.push([line_number, l1, l2])
|
|
end
|
|
line_number = line_number + 1
|
|
break if (l1 == nil && l2 == nil)
|
|
end
|
|
end
|
|
end
|
|
isDifferent = !diffs.empty?
|
|
diffOut = diffs.map {
|
|
| diff |
|
|
"@@ -\#{diff[0]},1 +\#{diff[0]},1 @@\\n" +
|
|
(diff[1] ? "-\#{diff[1]}" : "") +
|
|
(diff[2] ? "+\#{diff[2]}" : "")
|
|
}.join("")
|
|
END_FALLBACK_DIFF
|
|
end
|
|
|
|
def runDiff(expected, output)
|
|
<<-END_RUN_DIFF
|
|
diffOut, diffStatus = Open3.capture2("diff",
|
|
"--strip-trailing-cr",
|
|
"-u",
|
|
"#{expected}",
|
|
"#{output}");
|
|
isDifferent = !diffStatus.success?
|
|
END_RUN_DIFF
|
|
end
|
|
|
|
# Get a difference between two files, using diff where available, falling back
|
|
# on a limited comparison when diff is not available
|
|
def getDiff(expected, output)
|
|
if $hasDiff
|
|
runDiff(expected, output)
|
|
else
|
|
fallbackDiff(expected,output)
|
|
end
|
|
end
|
|
|
|
# Error handler for tests that diff their output with some expectation.
|
|
def diffErrorHandler(expectedFilename)
|
|
Proc.new {
|
|
| outp, plan |
|
|
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
|
|
|
|
outp.puts "if !success(status)\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code \#{status.exitstatus}\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "elsif File.exists?(\"../#{Shellwords.shellescape(expectedFilename)}\")\n"
|
|
outp.puts getDiff("../#{Shellwords.shellescape(expectedFilename)}", outputFilename)
|
|
outp.puts " if isDifferent\n"
|
|
outp.puts " print " + prefixString("\"DIFF FAILURE!\\n\"", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("diffOut", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts " else"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts " end"
|
|
outp.puts "else\n"
|
|
outp.puts " print " + prefixString("\"NO EXPECTATION!\\n\"", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "end"
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that report error by saying "failed!". This is used by Mozilla
|
|
# tests.
|
|
def mozillaErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if !success(status)\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code \#{status.exitstatus}\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "elsif /failed!/i =~ out\n"
|
|
outp.puts " print " + prefixString("\"Detected failures:\\n\"", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "else\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that report error by saying "failed!", and are expected to
|
|
# fail. This is used by Mozilla tests.
|
|
def mozillaFailErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if !success(status)\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "elsif /failed!/i =~ out\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "else\n"
|
|
outp.puts " print " + prefixString("\"NOTICE: You made this test pass, but it was expected to fail\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that report error by saying "failed!", and are expected to have
|
|
# an exit code of 3.
|
|
def mozillaExit3ErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if success(status)\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Test expected to fail, but returned successfully\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "elsif status.exitstatus != 3"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code: \#{status.exitstatus}\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "elsif /failed!/i =~ out\n"
|
|
outp.puts " print " + prefixString("\"Detected failures:\\n\"", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "else\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
# Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
|
|
# This is used by Chakra tests.
|
|
def chakraPassFailErrorHandler
|
|
Proc.new {
|
|
| outp, plan |
|
|
outp.puts "if !success(status)\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("\"ERROR: Unexpected exit code \#{status.exitstatus}\\n\"", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "elsif /FAILED/i =~ out\n"
|
|
outp.puts " print " + prefixString("\"Detected failures:\\n\"", plan.name) + "\n"
|
|
outp.puts " print " + prefixString("out", plan.name) + "\n"
|
|
outp.puts " " + plan.failCommand
|
|
outp.puts "else\n"
|
|
outp.puts " " + plan.successCommand
|
|
outp.puts "end\n"
|
|
}
|
|
end
|
|
|
|
class Plan
|
|
attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler, :additionalEnv
|
|
attr_accessor :index
|
|
|
|
def initialize(directory, arguments, family, name, outputHandler, errorHandler)
|
|
@directory = directory
|
|
@arguments = arguments
|
|
@family = family
|
|
@name = name
|
|
@outputHandler = outputHandler
|
|
@errorHandler = errorHandler
|
|
@isSlow = !!$runCommandOptions[:isSlow]
|
|
@crashOK = !!$runCommandOptions[:crashOK]
|
|
if @crashOK
|
|
@outputHandler = noisyOutputHandler
|
|
end
|
|
@additionalEnv = []
|
|
end
|
|
|
|
def shellCommand
|
|
script = "out = nil\n"
|
|
script += "err = nil\n"
|
|
script += "status = nil\n"
|
|
script += "Dir.chdir(\"../#{Shellwords.shellescape(@directory.to_s)}\") do\n"
|
|
script += " env = {}\n"
|
|
($envVars + additionalEnv).each {
|
|
|var|
|
|
(key, value) = var.split(/=/, 2)
|
|
script += " env[\"#{key}\"] = \"#{value}\"\n"
|
|
}
|
|
script += " out, err, status = Open3.capture3(env, \n"
|
|
script += @arguments.map { | argument | " \"#{argument}\""}.join(",\n")
|
|
script += " )\n"
|
|
script += "end\n"
|
|
return script
|
|
end
|
|
|
|
def reproScriptHelper
|
|
script = "def error_script_contents\n"
|
|
script += " <<-END_OF_SCRIPT\n"
|
|
script += " require 'open3'\n"
|
|
script += " def success(status)\n"
|
|
script += " status.success?\n"
|
|
script += " end\n"
|
|
script += " script_location = File.expand_path(File.dirname(__FILE__))\n"
|
|
script += " Dir.chdir(\"\\\#{script_location}"
|
|
Pathname.new(@name).dirname.each_filename {
|
|
| pathComponent |
|
|
script += "/.."
|
|
}
|
|
script += "/.runner\") do\n"
|
|
script += " ENV[\"DYLD_FRAMEWORK_PATH\"] = \"#{$testingFrameworkPath.dirname}\"\n"
|
|
script += " ENV[\"JSCTEST_timeout\"] = \"#{ENV['JSCTEST_timeout']}\"\n"
|
|
script += " ENV[\"JSCTEST_hardTimeout\"] = \"#{ENV['JSCTEST_hardTimeout']}\"\n"
|
|
script += " ENV[\"JSCTEST_memoryLimit\"] = \"#{ENV['JSCTEST_memoryLimit']}\"\n"
|
|
|
|
script += " #{shellCommand}"
|
|
script += " print out\n"
|
|
script += " if (!success(status))\n"
|
|
script += " exit(1)\n"
|
|
script += " end\n"
|
|
script += " end\n"
|
|
script += " END_OF_SCRIPT\n"
|
|
script += "end\n"
|
|
return script
|
|
end
|
|
|
|
def reproScriptCommand
|
|
<<-END_REPRO_SCRIPT_COMMAND
|
|
File.open("#{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}", "w") do |scr|
|
|
scr.puts "\#{error_script_contents}"
|
|
end
|
|
END_REPRO_SCRIPT_COMMAND
|
|
end
|
|
|
|
def statusCommand(status_code)
|
|
# May be called in th rescue block, so status is not
|
|
# guaranteed to be set; if it isn't, set the exit code to
|
|
# something that's clearly invalid.
|
|
<<-END_STATUS_COMMAND
|
|
File.open("#{statusFile}", "w") { |f|
|
|
f.puts("#{$runUniqueId} \#{status.nil? ? 999999999 : status.exitstatus} #{status_code}")
|
|
}
|
|
END_STATUS_COMMAND
|
|
end
|
|
|
|
def failCommand
|
|
<<-END_FAIL_COMMAND
|
|
print "FAIL: #{Shellwords.shellescape(@name)}\n"
|
|
#{statusCommand(STATUS_FILE_FAIL)}
|
|
#{reproScriptCommand}
|
|
END_FAIL_COMMAND
|
|
end
|
|
|
|
def successCommand
|
|
if $progressMeter or $verbosity >= 2
|
|
<<-END_VERBOSE_SUCCESS_COMMAND
|
|
print "PASS: #{Shellwords.shellescape(@name)}\n"
|
|
#{statusCommand(STATUS_FILE_PASS)}
|
|
END_VERBOSE_SUCCESS_COMMAND
|
|
else
|
|
"#{statusCommand(STATUS_FILE_PASS)}\n"
|
|
end
|
|
end
|
|
|
|
def statusFile
|
|
"#{STATUS_FILE_PREFIX}#{@index}"
|
|
end
|
|
|
|
def writeRunScript(filename)
|
|
File.open(filename, "w") {
|
|
| outp |
|
|
outp.puts "print \"Running #{Shellwords.shellescape(@name)}\\n\""
|
|
outp.puts "#{reproScriptHelper}"
|
|
outp.puts "begin"
|
|
outp.puts "require 'open3'"
|
|
outp.puts "require 'fileutils'"
|
|
outp.puts "def success(status)"
|
|
outp.puts " status.success?"
|
|
outp.puts "end"
|
|
|
|
cmd = shellCommand
|
|
|
|
cmd += @outputHandler.call(@name)
|
|
|
|
if $verbosity >= 3
|
|
outp.puts "print \"#{Shellwords.shellescape(cmd)}\\n\""
|
|
end
|
|
outp.puts cmd
|
|
@errorHandler.call(outp, self)
|
|
outp.puts "rescue"
|
|
outp.puts " print \"FAIL: #{Shellwords.shellescape(@name)}\\n\""
|
|
outp.puts " #{statusCommand(STATUS_FILE_FAIL)}"
|
|
outp.puts "end"
|
|
}
|
|
end
|
|
end
|
|
|
|
def prepareShellTestRunner
|
|
File.open($runnerDir + "runscript", "w") {
|
|
| outp |
|
|
$runlist.each {
|
|
| plan |
|
|
outp.puts "ruby test_script_#{plan.index}"
|
|
}
|
|
}
|
|
`dos2unix #{$runnerDir + "runscript"}`
|
|
end
|
|
|
|
def output_target(outp, plan, prereqs)
|
|
index = plan.index
|
|
target = "test_done_#{index}"
|
|
outp.puts "#{target}: #{prereqs.join(" ")}"
|
|
outp.puts "\truby test_script_#{index}"
|
|
target
|
|
end
|
|
|
|
def prepareMakeTestRunner(remoteIndex)
|
|
serialPlans = {}
|
|
$serialRunlist.each { |p| serialPlans[p] = nil }
|
|
runPlans = []
|
|
serialRunPlans = []
|
|
$runlist.each {
|
|
| plan |
|
|
if !$remote or plan.index % $remoteHosts.length == remoteIndex
|
|
if serialPlans.has_key?(plan)
|
|
serialRunPlans << plan
|
|
else
|
|
runPlans << plan
|
|
end
|
|
end
|
|
}
|
|
|
|
File.open($runnerDir + "Makefile.#{remoteIndex}", "w") {
|
|
| outp |
|
|
if serialRunPlans.empty?
|
|
outp.puts("all: parallel")
|
|
else
|
|
serialPrereq = "test_done_#{serialRunPlans[-1].index}"
|
|
outp.puts("all: #{serialPrereq}")
|
|
prev_target = "parallel"
|
|
serialRunPlans.each {
|
|
| plan |
|
|
prev_target = output_target(outp, plan, [prev_target])
|
|
}
|
|
end
|
|
parallelTargets = runPlans.collect {
|
|
| plan |
|
|
output_target(outp, plan, [])
|
|
}
|
|
outp.puts("parallel: " + parallelTargets.join(" "))
|
|
}
|
|
end
|
|
|
|
def prepareRubyTestRunner
|
|
File.open($runnerDir + "runscript", "w") {
|
|
| outp |
|
|
$runlist.each {
|
|
| plan |
|
|
outp.puts "system \"ruby test_script_#{plan.index}\""
|
|
}
|
|
}
|
|
end
|
|
|
|
def testRunnerCommand(remoteIndex=0)
|
|
case $testRunnerType
|
|
when :shell
|
|
command = "sh runscript"
|
|
when :make
|
|
command = "make -j #{$numChildProcesses} -s -f Makefile.#{remoteIndex}"
|
|
when :ruby
|
|
command = "ruby runscript"
|
|
else
|
|
raise "Unknown test runner type: #{$testRunnerType.to_s}"
|
|
end
|
|
return command
|
|
end
|