2019-03-08 23:20:15 +00:00
#!/usr/bin/env python -u
# Copyright (C) 2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of Apple Inc. ("Apple") nor the names of
# its contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2021-07-04 21:06:12 +00:00
import collections
2019-03-08 23:20:15 +00:00
import sys
import argparse
import json
2020-06-27 22:18:41 +00:00
import itertools
2020-06-13 20:20:34 +00:00
from webkitpy.benchmark_runner.benchmark_results import BenchmarkResults
2020-06-27 22:18:41 +00:00
from webkitpy.benchmark_runner.benchmark_json_merge import mergeJSONs
2019-03-08 23:20:15 +00:00
try:
from scipy import stats
except:
2021-01-05 17:59:56 +00:00
print("ERROR: scipy package is not installed. Run `pip install scipy`")
2019-03-08 23:20:15 +00:00
sys.exit(1)
try:
import numpy
except:
2021-01-05 17:59:56 +00:00
print("ERROR: numpy package is not installed. Run `pip install numpy`")
2019-03-08 23:20:15 +00:00
sys.exit(1)
def readJSONFile(path):
with open(path, 'r') as contents:
2020-06-13 20:20:34 +00:00
result = json.loads(contents.read())
if 'debugOutput' in result:
del result['debugOutput']
return result
2019-03-08 23:20:15 +00:00
Speedometer2 = "Speedometer2"
JetStream2 = "JetStream2"
PLT5 = "PLT5"
2021-07-04 21:06:12 +00:00
CompetitivePLT = "CompetitivePLT"
2021-01-08 00:45:26 +00:00
PLUM3 = "PLUM3"
2019-04-06 02:05:11 +00:00
MotionMark = "MotionMark"
MotionMark1_1 = "MotionMark-1.1"
2020-06-13 20:20:34 +00:00
MotionMark1_1_1 = "MotionMark-1.1.1"
unitMarker = "__unit__"
def speedometer2Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "ms"
for test in breakdown._results["Speedometer-2"]["tests"].keys():
result[test] = breakdown._results["Speedometer-2"]["tests"][test]["metrics"]["Time"]["Total"]["current"]
return result
def jetStream2Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "pts"
for test in breakdown._results["JetStream2.0"]["tests"].keys():
result[test] = breakdown._results["JetStream2.0"]["tests"][test]["metrics"]["Score"][None]["current"]
return result
def motionMarkBreakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "pts"
if detectMotionMark(jsonObject):
name = "MotionMark"
elif detectMotionMark1_1(jsonObject):
name = "MotionMark-1.1"
else:
name = "MotionMark-1.1.1"
for test in breakdown._results[name]["tests"].keys():
result[test] = breakdown._results[name]["tests"][test]["metrics"]["Score"][None]["current"]
return result
def plt5Breakdown(jsonObject):
nameMapping = {}
for mappings in jsonObject["urls"]:
for key in mappings.keys():
nameMapping[key] = mappings[key]
result = {}
result[unitMarker] = "ms"
for test in jsonObject["iterations"][0]["warm"].keys():
if test == "Geometric":
continue
result["warm--" + nameMapping[test]] = []
result["cold--" + nameMapping[test]] = []
for payload in jsonObject["iterations"]:
warmTests = payload["warm"]
coldTests = payload["cold"]
for test in warmTests.keys():
if test == "Geometric":
continue
result["warm--" + nameMapping[test]].append(warmTests[test]["Geometric"])
result["cold--" + nameMapping[test]].append(coldTests[test]["Geometric"])
return result
2021-07-04 21:06:12 +00:00
def competitivePLTBreakdown(jsonObject):
result = collections.defaultdict(list)
result[unitMarker] = "sec"
safari_results = jsonObject.get('Safari', {})
cold_results = safari_results.get('cold', {})
warm_results = safari_results.get('warm', {})
cold_link_results = cold_results.get('add-and-click-link', {})
warm_link_results = warm_results.get('add-and-click-link', {})
for site_to_times in cold_link_results.values():
for site, times in site_to_times.items():
result["cold--fmp--" + site].append(times['first_meaningful_paint'])
result["cold--load-end--" + site].append(times['load_end'])
for site_to_times in warm_link_results.values():
for site, times in site_to_times.items():
result["warm--fmp--" + site].append(times['first_meaningful_paint'])
result["warm--load-end--" + site].append(times['load_end'])
return result
2021-01-08 00:45:26 +00:00
def plum3Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "B"
for test in breakdown._results["PLUM3-PhysFootprint"]["tests"].keys():
result[test] = breakdown._results["PLUM3-PhysFootprint"]["tests"][test]["metrics"]["Allocations"]["Geometric"]["current"]
return result
2020-06-13 20:20:34 +00:00
def displayStr(value):
return "{:.6f}".format(float(value))
2020-06-16 23:29:21 +00:00
def computeMultipleHypothesesSignificance(a, b):
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
# This is using the Benjamini-Hochberg procedure based on False Discovery Rate
# for computing signifcance in multiple hypothesis testing
# Read more here:
# - https://en.wikipedia.org/wiki/False_discovery_rate
# - https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
# This is best used for independent variables. We know subtests aren't
# fully independent, this it's a reasonable approximation.
# We use this instead of Bonferroni because we control for almost the same
# false positive error rate (marking as signficant when it's not), but with a much
# lower false negative error rate (not marking something as signficant when it is).
sortedPValues = []
reversePValueMap = {}
for key in a.keys():
if key == unitMarker:
continue
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
sortedPValues.append(pValue)
if pValue not in reversePValueMap:
reversePValueMap[pValue] = []
reversePValueMap[pValue].append(key)
sortedPValues.sort()
assert sortedPValues[0] <= sortedPValues[-1]
2020-06-16 23:29:21 +00:00
isSignificant = False
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
result = {}
rank = float(len(sortedPValues))
for pValue in reversed(sortedPValues):
assert rank >= 1.0
threshold = (rank * .05) / float(len(sortedPValues))
if pValue <= threshold:
2020-06-16 23:29:21 +00:00
isSignificant = True
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
assert len(reversePValueMap[pValue]) > 0
for test in reversePValueMap[pValue]:
2020-06-16 23:29:21 +00:00
result[test] = isSignificant
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
rank = rank - 1.0
return result
2020-06-13 20:20:34 +00:00
def dumpBreakdowns(a, b):
nameLength = len("subtest")
aLength = len(a[unitMarker])
bLength = len(a[unitMarker])
ratioLength = len("b / a")
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
pValueHeader = "pValue (significance using False Discovery Rate)"
2020-06-13 20:20:34 +00:00
pLength = len(pValueHeader)
2020-06-16 23:29:21 +00:00
isSignificant = computeMultipleHypothesesSignificance(a, b)
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
2020-06-13 20:20:34 +00:00
for key in a.keys():
if key == unitMarker:
continue
nameLength = max(nameLength, len(key))
aLength = max(aLength, len(displayStr(numpy.mean(a[key]))))
bLength = max(bLength, len(displayStr(numpy.mean(b[key]))))
ratioLength = max(ratioLength, len(displayStr(numpy.mean(b[key]) / numpy.mean(a[key]))))
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
significantStr = ""
2020-06-16 23:29:21 +00:00
if isSignificant[key]:
2020-06-13 20:20:34 +00:00
significantStr = " (significant)"
pLength = max(pLength, len(displayStr(pValue)) + len(significantStr))
aLength += 2
bLength += 2
nameLength += 2
ratioLength += 2
pLength += 2
strings = []
strings.append("|{key:^{nameLength}}|{aScore:^{aLength}} |{bScore:^{bLength}} |{compare:^{ratioLength}}|{pMarker:^{pLength}}|".format(key="subtest", aScore=a[unitMarker], bScore=b[unitMarker], nameLength=nameLength, aLength=aLength, bLength=bLength , compare="b / a", ratioLength=ratioLength, pMarker=pValueHeader, pLength=pLength))
for key in a.keys():
if key == unitMarker:
continue
aScore = numpy.mean(a[key])
bScore = numpy.mean(b[key])
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
significantStr = ""
2020-06-16 23:29:21 +00:00
if isSignificant[key]:
2020-06-13 20:20:34 +00:00
significantStr = " (significant)"
strings.append("| {key:{nameLength}}|{aScore:{aLength}} |{bScore:{bLength}} |{compare:{ratioLength}}| {pValue:<{pLength}}|".format(key=key, aScore=displayStr(aScore), bScore=displayStr(bScore), nameLength=nameLength - 1, aLength=aLength, bLength=bLength, ratioLength=ratioLength, compare=displayStr(bScore / aScore), pValue = displayStr(pValue) + significantStr, pLength=pLength - 1))
maxLen = 0
for s in strings:
maxLen = max(maxLen, len(s))
verticalSeparator = "-" * maxLen
strings.insert(0, verticalSeparator)
strings.insert(2, verticalSeparator)
strings.append(verticalSeparator)
2021-01-05 17:59:56 +00:00
print("\n")
2020-06-13 20:20:34 +00:00
for s in strings:
print(s)
2021-01-05 17:59:56 +00:00
print("\n")
2020-06-13 20:20:34 +00:00
def writeCSV(a, b, fileName):
strings = []
result = ""
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
result += "test_name, {}, {}, b_divided_by_a, pValue, is_significant_using_False_Discovery_Rate\n".format("a_in_" + a[unitMarker], "b_in_" + b[unitMarker])
2020-06-13 20:20:34 +00:00
2020-06-16 23:29:21 +00:00
isSignificant = computeMultipleHypothesesSignificance(a, b)
2020-06-13 20:20:34 +00:00
for key in a.keys():
if key == unitMarker:
continue
aScore = numpy.mean(a[key])
bScore = numpy.mean(b[key])
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
significantStr = "No"
2020-06-16 23:29:21 +00:00
if isSignificant[key]:
Multiple hypothesis testing should use False Discovery Rate instead of Bonferroni
https://bugs.webkit.org/show_bug.cgi?id=213219
Reviewed by Mark Lam.
My previous patch here used Bonferroni to do multiple hypotheses correction.
Bonferroni does a good job at reducing false positive rate (type I error,
saying something is significant when it's not, e.g, rejecting the null
hypothesis when we shouldn't), but at the expense of also having a high
false negative rate (type II error, saying something is not significant when
it is, e.g, not rejecting the null hypothesis when we should).
After doing a bit more reading, it seems like using a technique called False
Discovery Rate (FDR) is better. It still does a good job at controlling type
I error rate, but without sacrificing a high type II error rate. FDR is based
on a technique for "estimating the proportion of wrongly rejected null hypotheses" [2].
This patch uses Benjamini and Hochberg estimation of FDR, which relies
on hypotheses being independent. We know subtests in a benchmark aren't
fully independent variables, but it's a good enough approximation.
You can read more about FDR:
1. https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
2. https://besjournals.onlinelibrary.wiley.com/doi/10.1111/j.2041-210X.2010.00061.x
3. https://en.wikipedia.org/wiki/False_discovery_rate
* Scripts/compare-results:
(displayStr):
(computeMultipleHypothesesSignficance):
(dumpBreakdowns):
(writeCSV):
Canonical link: https://commits.webkit.org/226046@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@263114 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-06-16 21:09:29 +00:00
significantStr = "Yes"
2020-06-13 20:20:34 +00:00
result += "{},{},{},{},{},{}\n".format(key, displayStr(aScore), displayStr(bScore), displayStr(bScore / aScore), displayStr(pValue), significantStr)
f = open(fileName, "w")
f.write(result)
f.close()
2019-03-08 23:20:15 +00:00
def detectJetStream2(payload):
return "JetStream2.0" in payload
def JetStream2Results(payload):
assert detectJetStream2(payload)
js = payload["JetStream2.0"]
iterations = len(js["tests"]["gaussian-blur"]["metrics"]["Score"]["current"])
results = []
for i in range(iterations):
scores = []
for test in js["tests"].keys():
scores.append(js["tests"][test]["metrics"]["Score"]["current"][i])
geomean = stats.gmean(scores)
results.append(geomean)
return results
def detectSpeedometer2(payload):
return "Speedometer-2" in payload
def Speedometer2Results(payload):
assert detectSpeedometer2(payload)
results = []
for arr in payload["Speedometer-2"]["metrics"]["Score"]["current"]:
results.append(numpy.mean(arr))
return results
def detectPLT5(payload):
if "iterations" not in payload:
return False
iterations = payload["iterations"]
if not isinstance(iterations, list):
return False
if not len(iterations):
return False
if "cold" not in iterations[0]:
return False
if "warm" not in iterations[0]:
return False
if "Geometric" not in iterations[0]:
return False
return True
def PLT5Results(payload):
assert detectPLT5(payload)
results = []
for obj in payload["iterations"]:
results.append(obj["Geometric"])
return results
2021-07-04 21:06:12 +00:00
def detectCompetitivePLT(payload):
return 'add-and-click-link' in payload.get('Safari', {}).get('cold', {})
def CompetitivePLTResults(payload):
def calculate_time_for_run(run):
# We geomean all FMP and load_end times together to produce a result for the run.
fmp_vals = [obj['first_meaningful_paint'] for obj in run.values()]
load_end_vals = [obj['load_end'] for obj in run.values()]
return stats.gmean(fmp_vals + load_end_vals)
safari_results = payload.get('Safari', {})
cold_results = safari_results.get('cold', {})
warm_results = safari_results.get('warm', {})
cold_link_results = cold_results.get('add-and-click-link', {})
warm_link_results = warm_results.get('add-and-click-link', {})
cold_times = [calculate_time_for_run(run) for run in cold_link_results.values()]
warm_times = [calculate_time_for_run(run) for run in warm_link_results.values()]
return [stats.gmean((cold_time, warm_time)) for cold_time, warm_time in zip(cold_times, warm_times)]
2021-01-08 00:45:26 +00:00
def detectPLUM3(payload):
return "PLUM3-PhysFootprint" in payload
def PLUM3Results(payload):
assert detectPLUM3(payload)
breakdown = BenchmarkResults(payload)
return breakdown._results["PLUM3-PhysFootprint"]["metrics"]["Allocations"]["Arithmetic"]["current"]
2019-04-06 02:05:11 +00:00
def detectMotionMark(payload):
return "MotionMark" in payload
def detectMotionMark1_1(payload):
return "MotionMark-1.1" in payload
2020-06-13 20:20:34 +00:00
def detectMotionMark1_1_1(payload):
return "MotionMark-1.1.1" in payload
2019-04-06 02:05:11 +00:00
def motionMarkResults(payload):
2020-06-13 20:20:34 +00:00
assert detectMotionMark(payload) or detectMotionMark1_1(payload) or detectMotionMark1_1_1(payload)
2019-04-06 02:05:11 +00:00
if detectMotionMark(payload):
payload = payload["MotionMark"]
2020-06-13 20:20:34 +00:00
elif detectMotionMark1_1(payload):
2019-04-06 02:05:11 +00:00
payload = payload["MotionMark-1.1"]
2020-06-13 20:20:34 +00:00
else:
payload = payload["MotionMark-1.1.1"]
2021-01-05 17:59:56 +00:00
testNames = list(payload["tests"].keys())
2019-04-06 02:05:11 +00:00
numTests = len(payload["tests"][testNames[0]]["metrics"]["Score"]["current"])
results = []
for i in range(numTests):
scores = []
for test in testNames:
scores.append(payload["tests"][test]["metrics"]["Score"]["current"][i])
results.append(stats.gmean(scores))
return results
2019-03-08 23:20:15 +00:00
def detectBenchmark(payload):
if detectJetStream2(payload):
return JetStream2
if detectSpeedometer2(payload):
return Speedometer2
if detectPLT5(payload):
return PLT5
2021-07-04 21:06:12 +00:00
if detectCompetitivePLT(payload):
return CompetitivePLT
2021-01-08 00:45:26 +00:00
if detectPLUM3(payload):
return PLUM3
2019-04-06 02:05:11 +00:00
if detectMotionMark(payload):
return MotionMark
if detectMotionMark1_1(payload):
return MotionMark1_1
2020-06-13 20:20:34 +00:00
if detectMotionMark1_1_1(payload):
return MotionMark1_1
2019-03-08 23:20:15 +00:00
return None
def biggerIsBetter(benchmarkType):
if benchmarkType == JetStream2:
return True
if benchmarkType == Speedometer2:
return True
2019-04-06 02:05:11 +00:00
if benchmarkType == MotionMark:
return True
if benchmarkType == MotionMark1_1:
return True
2019-03-08 23:20:15 +00:00
if benchmarkType == PLT5:
return False
2021-07-04 21:06:12 +00:00
if benchmarkType == CompetitivePLT:
return False
2021-01-08 00:45:26 +00:00
if benchmarkType == PLUM3:
return False
2019-03-08 23:20:15 +00:00
2021-01-05 17:59:56 +00:00
print("Should not be reached.")
2019-03-08 23:20:15 +00:00
assert False
def ttest(benchmarkType, a, b):
# We use two-tailed Welch's
(tStatistic, pValue) = stats.ttest_ind(a, b, equal_var=False)
aMean = numpy.mean(a)
bMean = numpy.mean(b)
2021-01-05 17:59:56 +00:00
print("a mean = {:.5f}".format(aMean))
print("b mean = {:.5f}".format(bMean))
2019-03-08 23:20:15 +00:00
2021-01-05 17:59:56 +00:00
print("pValue = {:.10f}".format(pValue))
2019-03-08 23:20:15 +00:00
if biggerIsBetter(benchmarkType):
2021-01-05 17:59:56 +00:00
print("(Bigger means are better.)")
2019-03-08 23:20:15 +00:00
if aMean > bMean:
2021-01-05 17:59:56 +00:00
print("{:.3f} times worse".format((aMean / bMean)))
2019-03-08 23:20:15 +00:00
else:
2021-01-05 17:59:56 +00:00
print("{:.3f} times better".format((bMean / aMean)))
2019-03-08 23:20:15 +00:00
else:
2021-01-05 17:59:56 +00:00
print("(Smaller means are better.)")
2019-03-08 23:20:15 +00:00
if aMean > bMean:
2021-01-05 17:59:56 +00:00
print("{:.3f} times better".format((aMean / bMean)))
2019-03-08 23:20:15 +00:00
else:
2021-01-05 17:59:56 +00:00
print("{:.3f} times worse".format((bMean / aMean)))
2019-03-08 23:20:15 +00:00
if pValue <= 0.05:
2021-01-05 17:59:56 +00:00
print("Results ARE significant")
2019-03-08 23:20:15 +00:00
else:
2021-01-05 17:59:56 +00:00
print("Results ARE NOT significant")
2019-03-08 23:20:15 +00:00
def getOptions():
2020-06-27 22:18:41 +00:00
parser = argparse.ArgumentParser(description="Compare two WebKit benchmark results. Pass in at least two JSON result files to compare them. This script prints the pValue along with the magnitude of the change. If more than one JSON is passed as a/b they will be merged when computing the breakdown.")
2019-03-08 23:20:15 +00:00
parser.add_argument("-a",
type=str,
required=True,
2020-06-27 22:18:41 +00:00
nargs='+',
action="append",
help="a JSONs of a/b. Path to JSON results file. Takes multiple values and can be passed multiple times.")
2019-03-08 23:20:15 +00:00
parser.add_argument("-b",
type=str,
required=True,
2020-06-27 22:18:41 +00:00
nargs='+',
action="append",
help="b JSONs of a/b. Path to JSON results file. Takes multiple values and can be passed multiple times.")
2019-03-08 23:20:15 +00:00
2020-06-13 20:20:34 +00:00
parser.add_argument("--csv",
type=str,
required=False,
help="Path to write a csv file containing subtest breakdown.")
parser.add_argument("--breakdown", action="store_true",
default=False, help="Print a per subtest breakdown.")
2019-03-08 23:20:15 +00:00
return parser.parse_known_args()[0]
def main():
args = getOptions()
2020-06-27 22:18:41 +00:00
# Flatten the list of lists of JSON files.
a = itertools.chain.from_iterable(args.a)
b = itertools.chain.from_iterable(args.b)
a = mergeJSONs(list(map(readJSONFile, a)))
b = mergeJSONs(list(map(readJSONFile, b)))
2019-03-08 23:20:15 +00:00
typeA = detectBenchmark(a)
typeB = detectBenchmark(b)
if typeA != typeB:
2021-01-05 17:59:56 +00:00
print("-a and -b are not the same benchmark. a={} b={}".format(typeA, typeB))
2019-03-08 23:20:15 +00:00
sys.exit(1)
if not (typeA and typeB):
2021-01-05 17:59:56 +00:00
print("Unknown benchmark type. a={} b={}".format(typeA, typeB))
2019-03-08 23:20:15 +00:00
sys.exit(1)
if typeA == JetStream2:
2020-06-13 20:20:34 +00:00
if args.breakdown:
dumpBreakdowns(jetStream2Breakdown(a), jetStream2Breakdown(b))
2019-03-08 23:20:15 +00:00
ttest(typeA, JetStream2Results(a), JetStream2Results(b))
2020-06-13 20:20:34 +00:00
if args.csv:
writeCSV(jetStream2Breakdown(a), jetStream2Breakdown(b), args.csv)
2019-03-08 23:20:15 +00:00
elif typeA == Speedometer2:
2020-06-13 20:20:34 +00:00
if args.breakdown:
dumpBreakdowns(speedometer2Breakdown(a), speedometer2Breakdown(b))
2019-03-08 23:20:15 +00:00
ttest(typeA, Speedometer2Results(a), Speedometer2Results(b))
2020-06-13 20:20:34 +00:00
if args.csv:
writeCSV(speedometer2Breakdown(a), speedometer2Breakdown(b), args.csv)
elif typeA == MotionMark or typeA == MotionMark1_1 or typeA == MotionMark1_1_1:
if args.breakdown:
dumpBreakdowns(motionMarkBreakdown(a), motionMarkBreakdown(b))
2019-04-06 02:05:11 +00:00
ttest(typeA, motionMarkResults(a), motionMarkResults(b))
2020-06-13 20:20:34 +00:00
if args.csv:
writeCSV(motionMarkBreakdown(a), motionMarkBreakdown(b), args.csv)
2019-03-08 23:20:15 +00:00
elif typeA == PLT5:
2020-06-13 20:20:34 +00:00
if args.breakdown:
dumpBreakdowns(plt5Breakdown(a), plt5Breakdown(b))
2019-03-08 23:20:15 +00:00
ttest(typeA, PLT5Results(a), PLT5Results(b))
2020-06-13 20:20:34 +00:00
if args.csv:
writeCSV(plt5Breakdown(a), plt5Breakdown(b), args.csv)
2021-07-04 21:06:12 +00:00
elif typeA == CompetitivePLT:
if args.breakdown:
dumpBreakdowns(competitivePLTBreakdown(a), competitivePLTBreakdown(b))
ttest(typeA, CompetitivePLTResults(a), CompetitivePLTResults(b))
if args.csv:
writeCSV(competitivePLTBreakdown(a), competitivePLTBreakdown(b), args.csv)
2021-01-08 00:45:26 +00:00
elif typeA == PLUM3:
if args.breakdown:
dumpBreakdowns(plum3Breakdown(a), plum3Breakdown(b))
ttest(typeA, PLUM3Results(a), PLUM3Results(b))
if args.csv:
writeCSV(plum3Breakdown(a), plum3Breakdown(b), args.csv)
2019-03-08 23:20:15 +00:00
else:
2021-01-05 17:59:56 +00:00
print("Unknown benchmark type")
2019-03-08 23:20:15 +00:00
sys.exit(1)
2019-04-19 21:35:08 +00:00
2019-03-08 23:20:15 +00:00
if __name__ == "__main__":
main()