415 lines
13 KiB
C++
415 lines
13 KiB
C++
/*
|
|
* Copyright (C) 2021 Apple Inc. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
|
|
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include "config.h"
|
|
#include "VerifierSlotVisitor.h"
|
|
|
|
#include "BlockDirectoryInlines.h"
|
|
#include "ConservativeRoots.h"
|
|
#include "GCSegmentedArrayInlines.h"
|
|
#include "HeapCell.h"
|
|
#include "JSCInlines.h"
|
|
#include "VM.h"
|
|
#include "VerifierSlotVisitorInlines.h"
|
|
#include <wtf/StackTrace.h>
|
|
|
|
namespace JSC {
|
|
|
|
using MarkerData = VerifierSlotVisitor::MarkerData;
|
|
|
|
constexpr int maxMarkingStackFramesToCapture = 100;
|
|
|
|
MarkerData::MarkerData(ReferrerToken referrer, std::unique_ptr<StackTrace>&& stack)
|
|
: m_referrer(referrer)
|
|
, m_stack(WTFMove(stack))
|
|
{
|
|
}
|
|
|
|
VerifierSlotVisitor::MarkedBlockData::MarkedBlockData(MarkedBlock* block)
|
|
: m_block(block)
|
|
{
|
|
}
|
|
|
|
void VerifierSlotVisitor::MarkedBlockData::addMarkerData(unsigned atomNumber, MarkerData&& marker)
|
|
{
|
|
if (m_markers.isEmpty())
|
|
m_markers.grow(MarkedBlock::atomsPerBlock);
|
|
m_markers[atomNumber] = WTFMove(marker);
|
|
}
|
|
|
|
const MarkerData* VerifierSlotVisitor::MarkedBlockData::markerData(unsigned atomNumber) const
|
|
{
|
|
auto& marker = m_markers[atomNumber];
|
|
if (marker.stack())
|
|
return ▮
|
|
return nullptr;
|
|
}
|
|
|
|
VerifierSlotVisitor::PreciseAllocationData::PreciseAllocationData(PreciseAllocation* allocation)
|
|
: m_allocation(allocation)
|
|
{
|
|
}
|
|
|
|
const MarkerData* VerifierSlotVisitor::PreciseAllocationData::markerData() const
|
|
{
|
|
if (m_marker.stack())
|
|
return &m_marker;
|
|
return nullptr;
|
|
}
|
|
|
|
void VerifierSlotVisitor::PreciseAllocationData::addMarkerData(MarkerData&& marker)
|
|
{
|
|
m_marker = WTFMove(marker);
|
|
}
|
|
|
|
const MarkerData* VerifierSlotVisitor::OpaqueRootData::markerData() const
|
|
{
|
|
if (m_marker.stack())
|
|
return &m_marker;
|
|
return nullptr;
|
|
}
|
|
|
|
void VerifierSlotVisitor::OpaqueRootData::addMarkerData(MarkerData&& marker)
|
|
{
|
|
m_marker = WTFMove(marker);
|
|
}
|
|
|
|
VerifierSlotVisitor::VerifierSlotVisitor(Heap& heap)
|
|
: Base(heap, "Verifier", m_opaqueRootStorage)
|
|
{
|
|
m_needsExtraOpaqueRootHandling = true;
|
|
}
|
|
|
|
VerifierSlotVisitor::~VerifierSlotVisitor()
|
|
{
|
|
heap()->objectSpace().forEachBlock(
|
|
[&] (MarkedBlock::Handle* handle) {
|
|
handle->block().setVerifierMemo(nullptr);
|
|
});
|
|
}
|
|
|
|
void VerifierSlotVisitor::addParallelConstraintTask(RefPtr<SharedTask<void(AbstractSlotVisitor&)>> task)
|
|
{
|
|
m_constraintTasks.append(WTFMove(task));
|
|
}
|
|
|
|
NO_RETURN_DUE_TO_CRASH void VerifierSlotVisitor::addParallelConstraintTask(RefPtr<SharedTask<void(SlotVisitor&)>>)
|
|
{
|
|
RELEASE_ASSERT_NOT_REACHED();
|
|
}
|
|
|
|
void VerifierSlotVisitor::executeConstraintTasks()
|
|
{
|
|
while (!m_constraintTasks.isEmpty())
|
|
m_constraintTasks.takeFirst()->run(*this);
|
|
}
|
|
|
|
void VerifierSlotVisitor::append(const ConservativeRoots& conservativeRoots)
|
|
{
|
|
auto appendJSCellOrAuxiliary = [&] (HeapCell* heapCell) {
|
|
if (!heapCell)
|
|
return;
|
|
|
|
if (testAndSetMarked(heapCell))
|
|
return;
|
|
|
|
switch (heapCell->cellKind()) {
|
|
case HeapCell::JSCell:
|
|
case HeapCell::JSCellWithIndexingHeader: {
|
|
JSCell* jsCell = static_cast<JSCell*>(heapCell);
|
|
appendToMarkStack(jsCell);
|
|
return;
|
|
}
|
|
|
|
case HeapCell::Auxiliary:
|
|
return;
|
|
}
|
|
};
|
|
|
|
HeapCell** roots = conservativeRoots.roots();
|
|
size_t size = conservativeRoots.size();
|
|
for (size_t i = 0; i < size; ++i)
|
|
appendJSCellOrAuxiliary(roots[i]);
|
|
}
|
|
|
|
void VerifierSlotVisitor::appendToMarkStack(JSCell* cell)
|
|
{
|
|
ASSERT(!cell->isZapped());
|
|
m_collectorStack.append(cell);
|
|
}
|
|
|
|
void VerifierSlotVisitor::appendUnbarriered(JSCell* cell)
|
|
{
|
|
if (!cell)
|
|
return;
|
|
|
|
if (UNLIKELY(cell->isPreciseAllocation())) {
|
|
if (LIKELY(isMarked(cell->preciseAllocation(), cell)))
|
|
return;
|
|
} else {
|
|
MarkedBlock& block = cell->markedBlock();
|
|
if (LIKELY(isMarked(block, cell)))
|
|
return;
|
|
}
|
|
|
|
appendSlow(cell);
|
|
}
|
|
|
|
void VerifierSlotVisitor::appendHiddenUnbarriered(JSCell* cell)
|
|
{
|
|
appendUnbarriered(cell);
|
|
}
|
|
|
|
void VerifierSlotVisitor::didAddOpaqueRoot(void* opaqueRoot)
|
|
{
|
|
if (!Options::verboseVerifyGC())
|
|
return;
|
|
|
|
std::unique_ptr<OpaqueRootData>& data = m_opaqueRootMap.add(opaqueRoot, nullptr).iterator->value;
|
|
if (!data)
|
|
data = makeUnique<OpaqueRootData>();
|
|
data->addMarkerData({ referrer(), StackTrace::captureStackTrace(maxMarkingStackFramesToCapture, 1) });
|
|
}
|
|
|
|
void VerifierSlotVisitor::didFindOpaqueRoot(void* opaqueRoot)
|
|
{
|
|
RELEASE_ASSERT(m_context && m_context->isOpaqueRootContext());
|
|
RELEASE_ASSERT(!m_context->referrer());
|
|
m_context->setReferrer(ReferrerToken(OpaqueRoot, opaqueRoot));
|
|
}
|
|
|
|
void VerifierSlotVisitor::drain()
|
|
{
|
|
RELEASE_ASSERT(m_mutatorStack.isEmpty());
|
|
|
|
MarkStackArray& stack = m_collectorStack;
|
|
if (stack.isEmpty())
|
|
return;
|
|
|
|
stack.refill();
|
|
while (stack.canRemoveLast())
|
|
visitChildren(stack.removeLast());
|
|
}
|
|
|
|
void VerifierSlotVisitor::dump(PrintStream& out) const
|
|
{
|
|
RELEASE_ASSERT(m_mutatorStack.isEmpty());
|
|
out.print("Verifier collector stack: ", m_collectorStack.size());
|
|
}
|
|
|
|
void VerifierSlotVisitor::dumpMarkerData(HeapCell* cell)
|
|
{
|
|
auto markerDataForPreciseAllocation = [&] (PreciseAllocation& allocation) -> const MarkerData* {
|
|
auto iterator = m_preciseAllocationMap.find(&allocation);
|
|
if (iterator == m_preciseAllocationMap.end())
|
|
return nullptr;
|
|
return iterator->value->markerData();
|
|
};
|
|
|
|
auto markerDataForMarkedBlockCell = [&] (MarkedBlock& block, HeapCell* cell) -> const MarkerData* {
|
|
auto iterator = m_markedBlockMap.find(&block);
|
|
if (iterator == m_markedBlockMap.end())
|
|
return nullptr;
|
|
unsigned atomNumber = block.atomNumber(cell);
|
|
return iterator->value->markerData(atomNumber);
|
|
};
|
|
|
|
auto markerDataForOpaqueRoot = [&] (void* opaqueRoot) -> const MarkerData* {
|
|
auto iterator = m_opaqueRootMap.find(opaqueRoot);
|
|
if (iterator == m_opaqueRootMap.end())
|
|
return nullptr;
|
|
return iterator->value->markerData();
|
|
};
|
|
|
|
WTF::dataFile().flush();
|
|
|
|
void* opaqueRoot = nullptr;
|
|
do {
|
|
const MarkerData* markerData = nullptr;
|
|
|
|
if (cell) {
|
|
if (isJSCellKind(cell->cellKind()))
|
|
dataLogLn(JSValue(static_cast<JSCell*>(cell)));
|
|
|
|
bool isMarked = heap()->isMarked(cell);
|
|
const char* wasOrWasNot = isMarked ? "was" : "was NOT";
|
|
dataLogLn("In the real GC, cell ", RawPointer(cell), " ", wasOrWasNot, " marked.");
|
|
|
|
if (cell->isPreciseAllocation())
|
|
markerData = markerDataForPreciseAllocation(cell->preciseAllocation());
|
|
else
|
|
markerData = markerDataForMarkedBlockCell(cell->markedBlock(), cell);
|
|
if (!markerData) {
|
|
dataLogLn("Marker data is not available for cell ", RawPointer(cell));
|
|
break;
|
|
}
|
|
dataLog("In the verifier GC, cell ", RawPointer(cell), " was visited");
|
|
|
|
} else {
|
|
RELEASE_ASSERT(opaqueRoot);
|
|
|
|
bool containsOpaqueRoot = heap()->m_opaqueRoots.contains(opaqueRoot);
|
|
const char* wasOrWasNot = containsOpaqueRoot ? "was" : "was NOT";
|
|
dataLogLn("In the real GC, opaque root ", RawPointer(opaqueRoot), " ", wasOrWasNot, " added to the heap's opaque roots.");
|
|
|
|
markerData = markerDataForOpaqueRoot(opaqueRoot);
|
|
if (!markerData) {
|
|
dataLogLn("Marker data is not available for opaque root ", RawPointer(opaqueRoot));
|
|
break;
|
|
}
|
|
dataLog("In the verifier GC, opaque root ", RawPointer(opaqueRoot), " was added");
|
|
}
|
|
|
|
ReferrerToken referrer = markerData->referrer();
|
|
if (auto* referrerCell = referrer.asCell()) {
|
|
dataLogLn(" via cell ", RawPointer(referrerCell), " at:");
|
|
cell = referrerCell;
|
|
opaqueRoot = nullptr;
|
|
} else if (auto* referrerOpaqueRoot = referrer.asOpaqueRoot()) {
|
|
dataLogLn(" via opaque root ", RawPointer(referrerOpaqueRoot), " at:");
|
|
cell = nullptr;
|
|
opaqueRoot = referrerOpaqueRoot;
|
|
} else {
|
|
auto reason = referrer.asRootMarkReason();
|
|
if (reason != RootMarkReason::None)
|
|
dataLogLn(" from scan of ", reason, " roots at:");
|
|
else
|
|
dataLogLn(" at:");
|
|
cell = nullptr;
|
|
opaqueRoot = nullptr;
|
|
}
|
|
|
|
markerData->stack()->dump(WTF::dataFile(), " ");
|
|
dataLogLn();
|
|
|
|
} while (cell || opaqueRoot);
|
|
}
|
|
|
|
bool VerifierSlotVisitor::isFirstVisit() const
|
|
{
|
|
// In the regular GC, this return value is only used to control whether
|
|
// UnlinkedCodeBlocks will be aged (see UnlinkedCodeBlock::visitChildrenImpl()).
|
|
// For the verifier GC pass, we should always return false because we don't
|
|
// want to do the aging action again.
|
|
return false;
|
|
}
|
|
|
|
bool VerifierSlotVisitor::isMarked(const void* rawCell) const
|
|
{
|
|
HeapCell* cell = bitwise_cast<HeapCell*>(rawCell);
|
|
if (cell->isPreciseAllocation())
|
|
return isMarked(cell->preciseAllocation(), cell);
|
|
return isMarked(cell->markedBlock(), cell);
|
|
}
|
|
|
|
bool VerifierSlotVisitor::isMarked(PreciseAllocation& allocation, HeapCell*) const
|
|
{
|
|
return m_preciseAllocationMap.contains(&allocation);
|
|
}
|
|
|
|
bool VerifierSlotVisitor::isMarked(MarkedBlock& block, HeapCell* cell) const
|
|
{
|
|
auto entry = m_markedBlockMap.find(&block);
|
|
if (entry == m_markedBlockMap.end())
|
|
return false;
|
|
|
|
auto& data = entry->value;
|
|
unsigned atomNumber = block.atomNumber(cell);
|
|
return data->isMarked(atomNumber);
|
|
}
|
|
|
|
void VerifierSlotVisitor::markAuxiliary(const void* base)
|
|
{
|
|
HeapCell* cell = bitwise_cast<HeapCell*>(base);
|
|
|
|
ASSERT(cell->heap() == heap());
|
|
testAndSetMarked(cell);
|
|
}
|
|
|
|
bool VerifierSlotVisitor::mutatorIsStopped() const
|
|
{
|
|
return true;
|
|
}
|
|
|
|
bool VerifierSlotVisitor::testAndSetMarked(const void* rawCell)
|
|
{
|
|
HeapCell* cell = bitwise_cast<HeapCell*>(rawCell);
|
|
if (cell->isPreciseAllocation())
|
|
return testAndSetMarked(cell->preciseAllocation());
|
|
return testAndSetMarked(cell->markedBlock(), cell);
|
|
}
|
|
|
|
bool VerifierSlotVisitor::testAndSetMarked(PreciseAllocation& allocation)
|
|
{
|
|
std::unique_ptr<PreciseAllocationData>& data = m_preciseAllocationMap.add(&allocation, nullptr).iterator->value;
|
|
if (!data) {
|
|
data = makeUnique<PreciseAllocationData>(&allocation);
|
|
if (UNLIKELY(Options::verboseVerifyGC()))
|
|
data->addMarkerData({ referrer(), StackTrace::captureStackTrace(maxMarkingStackFramesToCapture, 2) });
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool VerifierSlotVisitor::testAndSetMarked(MarkedBlock& block, HeapCell* cell)
|
|
{
|
|
MarkedBlockData* data = block.verifierMemo<MarkedBlockData*>();
|
|
if (UNLIKELY(!data)) {
|
|
std::unique_ptr<MarkedBlockData>& entryData = m_markedBlockMap.add(&block, nullptr).iterator->value;
|
|
RELEASE_ASSERT(!entryData);
|
|
entryData = makeUnique<MarkedBlockData>(&block);
|
|
data = entryData.get();
|
|
block.setVerifierMemo(data);
|
|
}
|
|
|
|
unsigned atomNumber = block.atomNumber(cell);
|
|
bool alreadySet = data->testAndSetMarked(atomNumber);
|
|
if (!alreadySet && UNLIKELY(Options::verboseVerifyGC()))
|
|
data->addMarkerData(atomNumber, { referrer(), StackTrace::captureStackTrace(maxMarkingStackFramesToCapture, 2) });
|
|
return alreadySet;
|
|
}
|
|
|
|
void VerifierSlotVisitor::setMarkedAndAppendToMarkStack(JSCell* cell)
|
|
{
|
|
if (m_suppressVerifier)
|
|
return;
|
|
if (testAndSetMarked(cell))
|
|
return;
|
|
appendToMarkStack(cell);
|
|
}
|
|
|
|
void VerifierSlotVisitor::visitAsConstraint(const JSCell* cell)
|
|
{
|
|
visitChildren(cell);
|
|
}
|
|
|
|
void VerifierSlotVisitor::visitChildren(const JSCell* cell)
|
|
{
|
|
RELEASE_ASSERT(isMarked(cell));
|
|
cell->methodTable(vm())->visitChildren(const_cast<JSCell*>(cell), *this);
|
|
}
|
|
|
|
} // namespace JSC
|