253 lines
10 KiB
C++
253 lines
10 KiB
C++
/*
|
|
* Copyright (C) 2013-2021 Apple Inc. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
|
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
|
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include "config.h"
|
|
#include "FTLThunks.h"
|
|
|
|
#if ENABLE(FTL_JIT)
|
|
|
|
#include "AssemblyHelpersSpoolers.h"
|
|
#include "DFGOSRExitCompilerCommon.h"
|
|
#include "FTLOSRExitCompiler.h"
|
|
#include "FTLOperations.h"
|
|
#include "FTLSaveRestore.h"
|
|
#include "GPRInfo.h"
|
|
#include "LinkBuffer.h"
|
|
|
|
namespace JSC { namespace FTL {
|
|
|
|
using namespace DFG;
|
|
|
|
enum class FrameAndStackAdjustmentRequirement {
|
|
Needed,
|
|
NotNeeded
|
|
};
|
|
|
|
static MacroAssemblerCodeRef<JITThunkPtrTag> genericGenerationThunkGenerator(
|
|
VM& vm, FunctionPtr<CFunctionPtrTag> generationFunction, PtrTag resultTag, const char* name, unsigned extraPopsToRestore, FrameAndStackAdjustmentRequirement frameAndStackAdjustmentRequirement)
|
|
{
|
|
AssemblyHelpers jit(nullptr);
|
|
|
|
if (frameAndStackAdjustmentRequirement == FrameAndStackAdjustmentRequirement::Needed) {
|
|
// This needs to happen before we use the scratch buffer because this function also uses the scratch buffer.
|
|
adjustFrameAndStackInOSRExitCompilerThunk<FTL::JITCode>(jit, vm, JITType::FTLJIT);
|
|
}
|
|
|
|
// Note that the "return address" will be the ID that we pass to the generation function.
|
|
|
|
constexpr GPRReg stackPointerRegister = MacroAssembler::stackPointerRegister;
|
|
constexpr GPRReg framePointerRegister = MacroAssembler::framePointerRegister;
|
|
constexpr ptrdiff_t pushToSaveByteOffset = MacroAssembler::pushToSaveByteOffset();
|
|
ptrdiff_t stackMisalignment = pushToSaveByteOffset;
|
|
|
|
// Pretend that we're a C call frame.
|
|
jit.pushToSave(framePointerRegister);
|
|
jit.move(stackPointerRegister, framePointerRegister);
|
|
stackMisalignment += pushToSaveByteOffset;
|
|
|
|
// Now create ourselves enough stack space to give saveAllRegisters() a scratch slot.
|
|
unsigned numberOfRequiredPops = 0;
|
|
do {
|
|
stackMisalignment += pushToSaveByteOffset;
|
|
numberOfRequiredPops++;
|
|
} while (stackMisalignment % stackAlignmentBytes());
|
|
jit.subPtr(MacroAssembler::TrustedImm32(numberOfRequiredPops * pushToSaveByteOffset), stackPointerRegister);
|
|
|
|
ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(requiredScratchMemorySizeInBytes());
|
|
char* buffer = static_cast<char*>(scratchBuffer->dataBuffer());
|
|
|
|
saveAllRegisters(jit, buffer);
|
|
|
|
jit.loadPtr(framePointerRegister, GPRInfo::argumentGPR0);
|
|
jit.peek(
|
|
GPRInfo::argumentGPR1,
|
|
(stackMisalignment - pushToSaveByteOffset) / sizeof(void*));
|
|
jit.prepareCallOperation(vm);
|
|
MacroAssembler::Call functionCall = jit.call(OperationPtrTag);
|
|
|
|
// At this point we want to make a tail call to what was returned to us in the
|
|
// returnValueGPR. But at the same time as we do this, we must restore all registers.
|
|
// The way we will accomplish this is by arranging to have the tail call target in the
|
|
// return address "slot" (be it a register or the stack).
|
|
|
|
jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0);
|
|
|
|
// Prepare for tail call.
|
|
|
|
jit.loadPtr(MacroAssembler::Address(stackPointerRegister, numberOfRequiredPops * pushToSaveByteOffset), framePointerRegister);
|
|
|
|
// When we came in here, there was an additional thing pushed to the stack (extraPopsToRestore).
|
|
// Some clients want it popped before proceeding. Also add 1 for the pushToSave of the framePointerRegister.
|
|
numberOfRequiredPops += 1 + extraPopsToRestore;
|
|
jit.addPtr(MacroAssembler::TrustedImm32(numberOfRequiredPops * pushToSaveByteOffset), stackPointerRegister);
|
|
|
|
// Put the return address wherever the return instruction wants it. On all platforms, this
|
|
// ensures that the return address is out of the way of register restoration.
|
|
jit.restoreReturnAddressBeforeReturn(GPRInfo::regT0);
|
|
|
|
restoreAllRegisters(jit, buffer);
|
|
|
|
#if CPU(ARM64E)
|
|
jit.untagPtr(resultTag, AssemblyHelpers::linkRegister);
|
|
jit.validateUntaggedPtr(AssemblyHelpers::linkRegister);
|
|
jit.tagReturnAddress();
|
|
#else
|
|
UNUSED_PARAM(resultTag);
|
|
#endif
|
|
jit.ret();
|
|
|
|
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::FTLThunk);
|
|
patchBuffer.link(functionCall, generationFunction.retagged<OperationPtrTag>());
|
|
return FINALIZE_CODE(patchBuffer, JITThunkPtrTag, "%s", name);
|
|
}
|
|
|
|
MacroAssemblerCodeRef<JITThunkPtrTag> osrExitGenerationThunkGenerator(VM& vm)
|
|
{
|
|
unsigned extraPopsToRestore = 0;
|
|
return genericGenerationThunkGenerator(
|
|
vm, operationCompileFTLOSRExit, OSRExitPtrTag, "FTL OSR exit generation thunk", extraPopsToRestore, FrameAndStackAdjustmentRequirement::Needed);
|
|
}
|
|
|
|
MacroAssemblerCodeRef<JITThunkPtrTag> lazySlowPathGenerationThunkGenerator(VM& vm)
|
|
{
|
|
unsigned extraPopsToRestore = 1;
|
|
return genericGenerationThunkGenerator(
|
|
vm, operationCompileFTLLazySlowPath, JITStubRoutinePtrTag, "FTL lazy slow path generation thunk", extraPopsToRestore, FrameAndStackAdjustmentRequirement::NotNeeded);
|
|
}
|
|
|
|
static void registerClobberCheck(AssemblyHelpers& jit, RegisterSet dontClobber)
|
|
{
|
|
if (!Options::clobberAllRegsInFTLICSlowPath())
|
|
return;
|
|
|
|
RegisterSet clobber = RegisterSet::allRegisters();
|
|
clobber.exclude(RegisterSet::reservedHardwareRegisters());
|
|
clobber.exclude(RegisterSet::stackRegisters());
|
|
clobber.exclude(RegisterSet::calleeSaveRegisters());
|
|
clobber.exclude(dontClobber);
|
|
|
|
GPRReg someGPR = InvalidGPRReg;
|
|
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
|
|
if (!clobber.get(reg) || !reg.isGPR())
|
|
continue;
|
|
|
|
jit.move(AssemblyHelpers::TrustedImm32(0x1337beef), reg.gpr());
|
|
someGPR = reg.gpr();
|
|
}
|
|
|
|
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
|
|
if (!clobber.get(reg) || !reg.isFPR())
|
|
continue;
|
|
|
|
jit.move64ToDouble(someGPR, reg.fpr());
|
|
}
|
|
}
|
|
|
|
MacroAssemblerCodeRef<JITThunkPtrTag> slowPathCallThunkGenerator(VM& vm, const SlowPathCallKey& key)
|
|
{
|
|
AssemblyHelpers jit(nullptr);
|
|
jit.tagReturnAddress();
|
|
|
|
// We want to save the given registers at the given offset, then we want to save the
|
|
// old return address somewhere past that offset, and then finally we want to make the
|
|
// call.
|
|
|
|
size_t currentOffset = key.offset() + sizeof(void*);
|
|
|
|
#if CPU(X86_64)
|
|
currentOffset += sizeof(void*);
|
|
#endif
|
|
|
|
AssemblyHelpers::StoreRegSpooler storeSpooler(jit, MacroAssembler::stackPointerRegister);
|
|
|
|
for (MacroAssembler::RegisterID reg = MacroAssembler::firstRegister(); reg <= MacroAssembler::lastRegister(); reg = static_cast<MacroAssembler::RegisterID>(reg + 1)) {
|
|
if (!key.usedRegisters().get(reg))
|
|
continue;
|
|
storeSpooler.storeGPR({ reg, static_cast<ptrdiff_t>(currentOffset) });
|
|
currentOffset += sizeof(void*);
|
|
}
|
|
storeSpooler.finalizeGPR();
|
|
|
|
for (MacroAssembler::FPRegisterID reg = MacroAssembler::firstFPRegister(); reg <= MacroAssembler::lastFPRegister(); reg = static_cast<MacroAssembler::FPRegisterID>(reg + 1)) {
|
|
if (!key.usedRegisters().get(reg))
|
|
continue;
|
|
storeSpooler.storeFPR({ reg, static_cast<ptrdiff_t>(currentOffset) });
|
|
currentOffset += sizeof(double);
|
|
}
|
|
storeSpooler.finalizeFPR();
|
|
|
|
jit.preserveReturnAddressAfterCall(GPRInfo::nonArgGPR1);
|
|
jit.storePtr(GPRInfo::nonArgGPR1, AssemblyHelpers::Address(MacroAssembler::stackPointerRegister, key.offset()));
|
|
jit.prepareCallOperation(vm);
|
|
|
|
RegisterSet dontClobber = key.argumentRegisters();
|
|
if (!key.callTarget())
|
|
dontClobber.set(GPRInfo::nonArgGPR0);
|
|
registerClobberCheck(jit, WTFMove(dontClobber));
|
|
|
|
AssemblyHelpers::Call call;
|
|
if (key.callTarget())
|
|
call = jit.call(OperationPtrTag);
|
|
else
|
|
jit.call(CCallHelpers::Address(GPRInfo::nonArgGPR0, key.indirectOffset()), OperationPtrTag);
|
|
|
|
jit.loadPtr(AssemblyHelpers::Address(MacroAssembler::stackPointerRegister, key.offset()), GPRInfo::nonPreservedNonReturnGPR);
|
|
jit.restoreReturnAddressBeforeReturn(GPRInfo::nonPreservedNonReturnGPR);
|
|
|
|
AssemblyHelpers::LoadRegSpooler loadSpooler(jit, MacroAssembler::stackPointerRegister);
|
|
|
|
for (MacroAssembler::FPRegisterID reg = MacroAssembler::lastFPRegister(); ; reg = static_cast<MacroAssembler::FPRegisterID>(reg - 1)) {
|
|
if (key.usedRegisters().get(reg)) {
|
|
currentOffset -= sizeof(double);
|
|
loadSpooler.loadFPR({ reg, static_cast<ptrdiff_t>(currentOffset) });
|
|
}
|
|
if (reg == MacroAssembler::firstFPRegister())
|
|
break;
|
|
}
|
|
loadSpooler.finalizeFPR();
|
|
|
|
for (MacroAssembler::RegisterID reg = MacroAssembler::lastRegister(); ; reg = static_cast<MacroAssembler::RegisterID>(reg - 1)) {
|
|
if (key.usedRegisters().get(reg)) {
|
|
currentOffset -= sizeof(void*);
|
|
loadSpooler.loadGPR({ reg, static_cast<ptrdiff_t>(currentOffset) });
|
|
}
|
|
if (reg == MacroAssembler::firstRegister())
|
|
break;
|
|
}
|
|
loadSpooler.finalizeGPR();
|
|
|
|
jit.ret();
|
|
|
|
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::FTLThunk);
|
|
if (key.callTarget())
|
|
patchBuffer.link(call, key.callTarget());
|
|
return FINALIZE_CODE(patchBuffer, JITThunkPtrTag, "FTL slow path call thunk for %s", toCString(key).data());
|
|
}
|
|
|
|
} } // namespace JSC::FTL
|
|
|
|
#endif // ENABLE(FTL_JIT)
|
|
|