329 lines
11 KiB
C++
329 lines
11 KiB
C++
/*
|
|
* Copyright (C) 2017-2019 Apple Inc. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
|
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
|
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include "IsoHeapImpl.h"
|
|
#include "IsoTLSDeallocatorEntry.h"
|
|
#include "IsoSharedHeapInlines.h"
|
|
#include "IsoSharedPageInlines.h"
|
|
|
|
namespace bmalloc {
|
|
|
|
template<typename Config>
|
|
IsoHeapImpl<Config>::IsoHeapImpl()
|
|
: IsoHeapImplBase((*PerProcess<IsoTLSEntryHolder<IsoTLSDeallocatorEntry<Config>>>::get())->lock)
|
|
, m_inlineDirectory(*this)
|
|
, m_allocator(*this)
|
|
{
|
|
#if BUSE(LIBPAS)
|
|
#pragma clang diagnostic push
|
|
#pragma clang diagnostic ignored "-Wmissing-noreturn"
|
|
RELEASE_BASSERT(!"Should not be using IsoHeapImpl if BUSE(LIBPAS)");
|
|
#pragma clang diagnostic pop
|
|
#endif
|
|
}
|
|
|
|
template<typename Config>
|
|
EligibilityResult<Config> IsoHeapImpl<Config>::takeFirstEligible(const LockHolder& locker)
|
|
{
|
|
if (m_isInlineDirectoryEligibleOrDecommitted) {
|
|
EligibilityResult<Config> result = m_inlineDirectory.takeFirstEligible(locker);
|
|
if (result.kind == EligibilityKind::Full)
|
|
m_isInlineDirectoryEligibleOrDecommitted = false;
|
|
else
|
|
return result;
|
|
}
|
|
|
|
{
|
|
auto* cursor = m_firstEligibleOrDecommitedDirectory.get();
|
|
if (!cursor) {
|
|
// If nothing is eligible, it can only be because we have no directories. It wouldn't be the end
|
|
// of the world if we broke this invariant. It would only mean that didBecomeEligibleOrDecommited() would need
|
|
// a null check.
|
|
RELEASE_BASSERT(!m_headDirectory.get());
|
|
RELEASE_BASSERT(!m_tailDirectory.get());
|
|
} else {
|
|
auto* originalCursor = cursor;
|
|
BUNUSED(originalCursor);
|
|
for (; cursor; cursor = cursor->next) {
|
|
EligibilityResult<Config> result = cursor->payload.takeFirstEligible(locker);
|
|
// While iterating, m_firstEligibleOrDecommitedDirectory is never changed. We are holding a lock,
|
|
// and IsoDirectory::takeFirstEligible must not populate a new eligibile / decommitted pages.
|
|
BASSERT(m_firstEligibleOrDecommitedDirectory.get() == originalCursor);
|
|
if (result.kind != EligibilityKind::Full) {
|
|
m_directoryHighWatermark = std::max(m_directoryHighWatermark, cursor->index());
|
|
m_firstEligibleOrDecommitedDirectory = cursor;
|
|
return result;
|
|
}
|
|
}
|
|
m_firstEligibleOrDecommitedDirectory = nullptr;
|
|
}
|
|
}
|
|
|
|
auto* newDirectory = new IsoDirectoryPage<Config>(*this, m_nextDirectoryPageIndex++);
|
|
if (m_headDirectory.get()) {
|
|
m_tailDirectory->next = newDirectory;
|
|
m_tailDirectory = newDirectory;
|
|
} else {
|
|
RELEASE_BASSERT(!m_tailDirectory.get());
|
|
m_headDirectory = newDirectory;
|
|
m_tailDirectory = newDirectory;
|
|
}
|
|
m_directoryHighWatermark = newDirectory->index();
|
|
m_firstEligibleOrDecommitedDirectory = newDirectory;
|
|
EligibilityResult<Config> result = newDirectory->payload.takeFirstEligible(locker);
|
|
RELEASE_BASSERT(result.kind != EligibilityKind::Full);
|
|
return result;
|
|
}
|
|
|
|
template<typename Config>
|
|
void IsoHeapImpl<Config>::didBecomeEligibleOrDecommited(const LockHolder&, IsoDirectory<Config, numPagesInInlineDirectory>* directory)
|
|
{
|
|
RELEASE_BASSERT(directory == &m_inlineDirectory);
|
|
m_isInlineDirectoryEligibleOrDecommitted = true;
|
|
}
|
|
|
|
template<typename Config>
|
|
void IsoHeapImpl<Config>::didBecomeEligibleOrDecommited(const LockHolder&, IsoDirectory<Config, IsoDirectoryPage<Config>::numPages>* directory)
|
|
{
|
|
RELEASE_BASSERT(m_firstEligibleOrDecommitedDirectory);
|
|
auto* directoryPage = IsoDirectoryPage<Config>::pageFor(directory);
|
|
if (directoryPage->index() < m_firstEligibleOrDecommitedDirectory->index())
|
|
m_firstEligibleOrDecommitedDirectory = directoryPage;
|
|
}
|
|
|
|
template<typename Config>
|
|
void IsoHeapImpl<Config>::scavenge(Vector<DeferredDecommit>& decommits)
|
|
{
|
|
LockHolder locker(this->lock);
|
|
forEachDirectory(
|
|
locker,
|
|
[&] (auto& directory) {
|
|
directory.scavenge(locker, decommits);
|
|
});
|
|
m_directoryHighWatermark = 0;
|
|
}
|
|
|
|
inline size_t IsoHeapImplBase::freeableMemory()
|
|
{
|
|
return m_freeableMemory;
|
|
}
|
|
|
|
template<typename Config>
|
|
unsigned IsoHeapImpl<Config>::allocatorOffset()
|
|
{
|
|
return m_allocator->offset();
|
|
}
|
|
|
|
template<typename Config>
|
|
unsigned IsoHeapImpl<Config>::deallocatorOffset()
|
|
{
|
|
return (*PerProcess<IsoTLSEntryHolder<IsoTLSDeallocatorEntry<Config>>>::get())->offset();
|
|
}
|
|
|
|
template<typename Config>
|
|
unsigned IsoHeapImpl<Config>::numLiveObjects()
|
|
{
|
|
LockHolder locker(this->lock);
|
|
unsigned result = 0;
|
|
forEachLiveObject(
|
|
locker,
|
|
[&] (void*) {
|
|
result++;
|
|
});
|
|
return result;
|
|
}
|
|
|
|
template<typename Config>
|
|
unsigned IsoHeapImpl<Config>::numCommittedPages()
|
|
{
|
|
LockHolder locker(this->lock);
|
|
unsigned result = 0;
|
|
forEachCommittedPage(
|
|
locker,
|
|
[&] (IsoPage<Config>&) {
|
|
result++;
|
|
});
|
|
return result;
|
|
}
|
|
|
|
template<typename Config>
|
|
template<typename Func>
|
|
void IsoHeapImpl<Config>::forEachDirectory(const LockHolder&, const Func& func)
|
|
{
|
|
func(m_inlineDirectory);
|
|
for (IsoDirectoryPage<Config>* page = m_headDirectory.get(); page; page = page->next)
|
|
func(page->payload);
|
|
}
|
|
|
|
template<typename Config>
|
|
template<typename Func>
|
|
void IsoHeapImpl<Config>::forEachCommittedPage(const LockHolder& locker, const Func& func)
|
|
{
|
|
forEachDirectory(
|
|
locker,
|
|
[&] (auto& directory) {
|
|
directory.forEachCommittedPage(locker, func);
|
|
});
|
|
}
|
|
|
|
template<typename Config>
|
|
template<typename Func>
|
|
void IsoHeapImpl<Config>::forEachLiveObject(const LockHolder& locker, const Func& func)
|
|
{
|
|
forEachCommittedPage(
|
|
locker,
|
|
[&] (IsoPage<Config>& page) {
|
|
page.forEachLiveObject(locker, func);
|
|
});
|
|
for (unsigned index = 0; index < maxAllocationFromShared; ++index) {
|
|
void* pointer = m_sharedCells[index].get();
|
|
if (pointer && !(m_availableShared & (1U << index)))
|
|
func(pointer);
|
|
}
|
|
}
|
|
|
|
inline size_t IsoHeapImplBase::footprint()
|
|
{
|
|
#if ENABLE_PHYSICAL_PAGE_MAP
|
|
RELEASE_BASSERT(m_footprint == m_physicalPageMap.footprint());
|
|
#endif
|
|
return m_footprint;
|
|
}
|
|
|
|
inline void IsoHeapImplBase::didCommit(void* ptr, size_t bytes)
|
|
{
|
|
BUNUSED_PARAM(ptr);
|
|
m_footprint += bytes;
|
|
#if ENABLE_PHYSICAL_PAGE_MAP
|
|
m_physicalPageMap.commit(ptr, bytes);
|
|
#endif
|
|
}
|
|
|
|
inline void IsoHeapImplBase::didDecommit(void* ptr, size_t bytes)
|
|
{
|
|
BUNUSED_PARAM(ptr);
|
|
m_footprint -= bytes;
|
|
#if ENABLE_PHYSICAL_PAGE_MAP
|
|
m_physicalPageMap.decommit(ptr, bytes);
|
|
#endif
|
|
}
|
|
|
|
inline void IsoHeapImplBase::isNowFreeable(void* ptr, size_t bytes)
|
|
{
|
|
BUNUSED_PARAM(ptr);
|
|
m_freeableMemory += bytes;
|
|
}
|
|
|
|
inline void IsoHeapImplBase::isNoLongerFreeable(void* ptr, size_t bytes)
|
|
{
|
|
BUNUSED_PARAM(ptr);
|
|
m_freeableMemory -= bytes;
|
|
}
|
|
|
|
template<typename Config>
|
|
AllocationMode IsoHeapImpl<Config>::updateAllocationMode()
|
|
{
|
|
auto getNewAllocationMode = [&] {
|
|
// Exhaust shared free cells, which means we should start activating the fast allocation mode for this type.
|
|
if (!m_availableShared) {
|
|
m_lastSlowPathTime = std::chrono::steady_clock::now();
|
|
return AllocationMode::Fast;
|
|
}
|
|
|
|
switch (m_allocationMode) {
|
|
case AllocationMode::Shared:
|
|
// Currently in the shared allocation mode. Until we exhaust shared free cells, continue using the shared allocation mode.
|
|
// But if we allocate so many shared cells within very short period, we should use the fast allocation mode instead.
|
|
// This avoids the following pathological case.
|
|
//
|
|
// for (int i = 0; i < 1e6; ++i) {
|
|
// auto* ptr = allocate();
|
|
// ...
|
|
// free(ptr);
|
|
// }
|
|
if (m_numberOfAllocationsFromSharedInOneCycle <= IsoPage<Config>::numObjects)
|
|
return AllocationMode::Shared;
|
|
BFALLTHROUGH;
|
|
|
|
case AllocationMode::Fast: {
|
|
// The allocation pattern may change. We should check the allocation rate and decide which mode is more appropriate.
|
|
// If we don't go to the allocation slow path during ~1 seconds, we think the allocation becomes quiescent state.
|
|
auto now = std::chrono::steady_clock::now();
|
|
if ((now - m_lastSlowPathTime) < std::chrono::seconds(1)) {
|
|
m_lastSlowPathTime = now;
|
|
return AllocationMode::Fast;
|
|
}
|
|
|
|
m_numberOfAllocationsFromSharedInOneCycle = 0;
|
|
m_lastSlowPathTime = now;
|
|
return AllocationMode::Shared;
|
|
}
|
|
|
|
case AllocationMode::Init:
|
|
m_lastSlowPathTime = std::chrono::steady_clock::now();
|
|
return AllocationMode::Shared;
|
|
}
|
|
|
|
return AllocationMode::Shared;
|
|
};
|
|
AllocationMode allocationMode = getNewAllocationMode();
|
|
m_allocationMode = allocationMode;
|
|
return allocationMode;
|
|
}
|
|
|
|
template<typename Config>
|
|
void* IsoHeapImpl<Config>::allocateFromShared(const LockHolder&, bool abortOnFailure)
|
|
{
|
|
static constexpr bool verbose = false;
|
|
|
|
unsigned indexPlusOne = __builtin_ffs(m_availableShared);
|
|
BASSERT(indexPlusOne);
|
|
unsigned index = indexPlusOne - 1;
|
|
void* result = m_sharedCells[index].get();
|
|
if (result) {
|
|
if (verbose)
|
|
fprintf(stderr, "%p: allocated %p from shared again of size %u\n", this, result, Config::objectSize);
|
|
} else {
|
|
constexpr unsigned objectSizeWithHeapImplPointer = Config::objectSize + sizeof(uint8_t);
|
|
result = IsoSharedHeap::get()->allocateNew<objectSizeWithHeapImplPointer>(abortOnFailure);
|
|
if (!result)
|
|
return nullptr;
|
|
if (verbose)
|
|
fprintf(stderr, "%p: allocated %p from shared of size %u\n", this, result, Config::objectSize);
|
|
BASSERT(index < IsoHeapImplBase::maxAllocationFromShared);
|
|
*indexSlotFor<Config>(result) = index;
|
|
m_sharedCells[index] = bitwise_cast<uint8_t*>(result);
|
|
}
|
|
BASSERT(result);
|
|
m_availableShared &= ~(1U << index);
|
|
++m_numberOfAllocationsFromSharedInOneCycle;
|
|
return result;
|
|
}
|
|
|
|
} // namespace bmalloc
|
|
|