289 lines
7.4 KiB
Bash
Executable File
289 lines
7.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright (C) 2020 Apple Inc. All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
#
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
|
|
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
|
|
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
set -eo pipefail
|
|
|
|
API_BENCH_EXECUTABLE=$(basename "$0")
|
|
|
|
pushd "$(dirname "$0")" &> /dev/null
|
|
API_BENCH_DIR=$PWD
|
|
popd &> /dev/null
|
|
|
|
CURRENT_BENCHMARK_ID=0
|
|
BENCHMARKS=()
|
|
BUILD_DIRECTORIES=()
|
|
REFERENCES=("X")
|
|
CURRENT_API=()
|
|
UPCOMING_API=()
|
|
ARCHS=${ARCHS:-$(uname -m)}
|
|
CONFIGURATION=${CONFIGURATION:-Release}
|
|
|
|
shouldBuild=true
|
|
runReferences=true
|
|
verbose=false
|
|
iterations=5
|
|
|
|
log() {
|
|
if [ "$verbose" == true ]; then
|
|
echo "$*"
|
|
fi
|
|
}
|
|
|
|
printUsage() {
|
|
cat <<EOF
|
|
usage: $API_BENCH_EXECUTABLE [-h|--help] [--no-build] [--no-reference(s)] [-v|--verbose] [--iterations <n=$iterations>] <build_directory1> [<build_directory2> ...]
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
collectFlags() {
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-h|--help)
|
|
printUsage
|
|
;;
|
|
--no-build)
|
|
shouldBuild=false
|
|
;;
|
|
--no-reference|--no-references)
|
|
runReferences=false
|
|
;;
|
|
-v|--verbose)
|
|
verbose=true
|
|
;;
|
|
--iterations)
|
|
shift
|
|
iterations=$1
|
|
;;
|
|
*)
|
|
BUILD_DIRECTORIES=("$@")
|
|
break 2
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [ ${#BUILD_DIRECTORIES[@]} -lt 1 ]; then
|
|
printUsage
|
|
exit 1
|
|
fi
|
|
|
|
for i in "${!BUILD_DIRECTORIES[@]}"; do
|
|
local buildDirectory
|
|
buildDirectory=${BUILD_DIRECTORIES[$i]}
|
|
|
|
if ! [ -d "$buildDirectory" ]; then
|
|
echo "No such file or directory: $buildDirectory"
|
|
exit 1
|
|
fi
|
|
|
|
pushd "$buildDirectory" &> /dev/null
|
|
BUILD_DIRECTORIES[$i]=$PWD
|
|
popd &> /dev/null
|
|
done
|
|
}
|
|
|
|
collectBenchmarkDir() {
|
|
local benchmarkVar=$1
|
|
local benchmarkDir=$2
|
|
benchmarks=("$API_BENCH_DIR"/"$benchmarkDir"/*)
|
|
for benchmark in "${benchmarks[@]}"; do
|
|
local benchmarkID=$CURRENT_BENCHMARK_ID
|
|
CURRENT_BENCHMARK_ID=$((CURRENT_BENCHMARK_ID+1))
|
|
pushd "$benchmark" &> /dev/null
|
|
BENCHMARKS[$benchmarkID]=$PWD
|
|
eval "${benchmarkVar}[\$benchmarkID]=$benchmarkID"
|
|
popd &> /dev/null
|
|
done
|
|
}
|
|
|
|
collectBenchmarks() {
|
|
if [ "$runReferences" == true ]; then
|
|
collectBenchmarkDir "REFERENCES" References
|
|
fi
|
|
collectBenchmarkDir "CURRENT_API" CurrentAPI
|
|
collectBenchmarkDir "UPCOMING_API" UpcomingAPI
|
|
}
|
|
|
|
forEachBenchmark() {
|
|
local functor
|
|
functor="$1"
|
|
declare "$functor"
|
|
for benchmarkID in "${!BENCHMARKS[@]}"; do
|
|
local benchmark
|
|
benchmark=${BENCHMARKS[$benchmarkID]}
|
|
pushd "$benchmark" &> /dev/null
|
|
BENCHMARK_ID=$benchmarkID
|
|
BENCHMARK_NAME=$(basename "$benchmark")
|
|
"$functor"
|
|
popd &> /dev/null
|
|
done
|
|
}
|
|
|
|
buildBenchmark() {
|
|
project=$BENCHMARK_NAME.xcodeproj
|
|
|
|
if ! [ -e "$project" ]; then
|
|
return
|
|
fi
|
|
|
|
log "Building $BENCHMARK_NAME... (FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY')"
|
|
xcodebuild -project "$project" -target "$BENCHMARK_NAME" -configuration "$CONFIGURATION" build "FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY'" "ARCHS='$ARCHS'"
|
|
}
|
|
|
|
build() {
|
|
BUILD_DIRECTORY=$1
|
|
forEachBenchmark "buildBenchmark"
|
|
}
|
|
|
|
runJSBenchmark() {
|
|
"$DYLD_FRAMEWORK_PATH/jsc" "$BENCHMARK_NAME.js"
|
|
}
|
|
|
|
runNativeBenchmark() {
|
|
pushd "build/$CONFIGURATION" &> /dev/null
|
|
"./$BENCHMARK_NAME"
|
|
local exitCode=$?
|
|
if [ $exitCode -ne 0 ]; then
|
|
echo "Benchmark exited with an error: $PWD/$BENCHMARK_NAME exited with $exitCode" 1>&2
|
|
fi
|
|
popd &> /dev/null
|
|
return $exitCode
|
|
}
|
|
|
|
runBenchmarkIteration() {
|
|
if [ -e "build" ]; then
|
|
runNativeBenchmark
|
|
else
|
|
runJSBenchmark
|
|
fi
|
|
}
|
|
|
|
runBenchmark() {
|
|
log "Running $BENCHMARK_NAME..."
|
|
|
|
local result
|
|
result=$(runBenchmarkIteration)
|
|
if [ $? -ne 0 ]; then
|
|
exit 1
|
|
fi
|
|
RESULTS[$((BENCHMARK_ID * iterations + ITERATION))]=$result
|
|
log "Finished in ${result}ms"
|
|
}
|
|
|
|
computeAverage() {
|
|
local benchmarkID benchmark sum avg
|
|
benchmarkID=$1
|
|
benchmark=${BENCHMARKS[$benchmarkID]}
|
|
sum=0
|
|
for iteration in $(seq 0 1 $((iterations - 1))); do
|
|
local result
|
|
result=${RESULTS[$((benchmarkID * iterations + iteration))]}
|
|
sum=$((sum + result))
|
|
done
|
|
bc <<< "$sum.0 / $iterations.0"
|
|
}
|
|
|
|
printAverage() {
|
|
local benchmarkID benchmark name avg
|
|
benchmarkID=$1
|
|
avg=$2
|
|
benchmark=${BENCHMARKS[$benchmarkID]}
|
|
name=$(basename "$benchmark")
|
|
echo "$name: ${avg}ms"
|
|
}
|
|
|
|
printBenchmark() {
|
|
printAverage "$@" "$(computeAverage "$@")"
|
|
}
|
|
|
|
printResults() {
|
|
local buildDirectory=$1
|
|
|
|
echo "Results for $buildDirectory ($iterations iterations)"
|
|
echo "================================================================================"
|
|
|
|
if [ "$runReferences" == true ]; then
|
|
echo ""
|
|
echo "References:"
|
|
echo "----------------------------------------"
|
|
for benchmarkID in "${REFERENCES[@]}"; do
|
|
printBenchmark "$benchmarkID"
|
|
done
|
|
fi
|
|
|
|
local score count
|
|
score=0
|
|
count=0
|
|
|
|
printCategory() {
|
|
local category=$1
|
|
|
|
echo ""
|
|
echo "$category:"
|
|
echo "----------------------------------------"
|
|
|
|
for benchmarkID in $(eval 'echo ${'"$category"'[@]}'); do
|
|
avg=$(computeAverage "$benchmarkID")
|
|
printAverage "$benchmarkID" "$avg"
|
|
count=$((count + 1))
|
|
score=$(echo "$score" "$avg" | awk '{print ($1 + log($2)) }')
|
|
done
|
|
}
|
|
printCategory "CURRENT_API"
|
|
printCategory "UPCOMING_API"
|
|
score=$(echo "$score" "$count" | awk '{ print(5000.0 / exp($1 / $2)) }')
|
|
|
|
echo ""
|
|
echo "Score: $score"
|
|
}
|
|
|
|
run() {
|
|
local buildDirectory
|
|
buildDirectory=$1
|
|
|
|
export DYLD_FRAMEWORK_PATH=$buildDirectory
|
|
for iteration in $(seq 0 1 $((iterations-1))); do
|
|
ITERATION=$iteration
|
|
forEachBenchmark "runBenchmark"
|
|
done
|
|
|
|
printResults "$buildDirectory"
|
|
}
|
|
|
|
|
|
main() {
|
|
collectFlags "$@"
|
|
collectBenchmarks
|
|
for buildDirectory in "${BUILD_DIRECTORIES[@]}"; do
|
|
if [ "$shouldBuild" == true ]; then
|
|
build "$buildDirectory"
|
|
fi
|
|
|
|
run "$buildDirectory"
|
|
done
|
|
}
|
|
|
|
main "$@"
|