haikuwebkit/Source/bmalloc/bmalloc/Packed.h

240 lines
6.7 KiB
C
Raw Permalink Normal View History

[bmalloc] IsoHeap's initial setup should be small https://bugs.webkit.org/show_bug.cgi?id=206214 Reviewed by Michael Saboff. Source/bmalloc: Keep IsoHeap related data structures small by using Packed technique. We start using IsoHeap for many classes, then it is important that we keep metadata for IsoHeap small because these data persistently exists. 1. We pass IsoHeapImpl<> as a parameter instead of holding it unnecessarily. 2. We make some of pointers Packed so that we can keep sizeof(IsoHeapImpl<Config>) small. 3. One of the drawback of PackedPtr is that loading and storing are not atomic. And we pass `const std::lock_guard<Mutex>&` to functions if functions need to be called with lock so that we ensure that PackedPtr are accessed only when lock is held correctly. * CMakeLists.txt: * bmalloc.xcodeproj/project.pbxproj: * bmalloc/Algorithm.h: (bmalloc::ctzConstexpr): (bmalloc::getLSBSetNonZeroConstexpr): * bmalloc/BPlatform.h: * bmalloc/DebugHeap.cpp: (bmalloc::DebugHeap::DebugHeap): * bmalloc/DebugHeap.h: * bmalloc/DeferredTrigger.h: * bmalloc/DeferredTriggerInlines.h: (bmalloc::DeferredTrigger<trigger>::didBecome): (bmalloc::DeferredTrigger<trigger>::handleDeferral): * bmalloc/Environment.cpp: (bmalloc::Environment::Environment): * bmalloc/Environment.h: * bmalloc/Gigacage.cpp: (bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks): * bmalloc/Heap.cpp: (bmalloc::Heap::freeableMemory): (bmalloc::Heap::markAllLargeAsEligibile): (bmalloc::Heap::decommitLargeRange): (bmalloc::Heap::scavenge): (bmalloc::Heap::scavengeToHighWatermark): * bmalloc/Heap.h: * bmalloc/HeapConstants.cpp: (bmalloc::HeapConstants::HeapConstants): * bmalloc/HeapConstants.h: * bmalloc/IsoAllocator.h: * bmalloc/IsoAllocatorInlines.h: (bmalloc::IsoAllocator<Config>::IsoAllocator): (bmalloc::IsoAllocator<Config>::allocate): (bmalloc::IsoAllocator<Config>::allocateSlow): (bmalloc::IsoAllocator<Config>::scavenge): * bmalloc/IsoDeallocatorInlines.h: (bmalloc::IsoDeallocator<Config>::scavenge): * bmalloc/IsoDirectory.h: * bmalloc/IsoDirectoryInlines.h: (bmalloc::passedNumPages>::IsoDirectory): (bmalloc::passedNumPages>::takeFirstEligible): (bmalloc::passedNumPages>::didBecome): (bmalloc::passedNumPages>::didDecommit): (bmalloc::passedNumPages>::scavengePage): (bmalloc::passedNumPages>::scavenge): (bmalloc::passedNumPages>::scavengeToHighWatermark): (bmalloc::passedNumPages>::forEachCommittedPage): * bmalloc/IsoHeapImpl.cpp: (bmalloc::IsoHeapImplBase::IsoHeapImplBase): * bmalloc/IsoHeapImpl.h: * bmalloc/IsoHeapImplInlines.h: (bmalloc::IsoHeapImpl<Config>::IsoHeapImpl): (bmalloc::IsoHeapImpl<Config>::takeFirstEligible): (bmalloc::IsoHeapImpl<Config>::didBecomeEligibleOrDecommited): (bmalloc::IsoHeapImpl<Config>::scavenge): (bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark): (bmalloc::IsoHeapImplBase::freeableMemory): (bmalloc::IsoHeapImpl<Config>::numLiveObjects): (bmalloc::IsoHeapImpl<Config>::numCommittedPages): (bmalloc::IsoHeapImpl<Config>::forEachDirectory): (bmalloc::IsoHeapImpl<Config>::forEachCommittedPage): (bmalloc::IsoHeapImpl<Config>::forEachLiveObject): (bmalloc::IsoHeapImplBase::footprint): (bmalloc::IsoHeapImplBase::didCommit): (bmalloc::IsoHeapImplBase::didDecommit): (bmalloc::IsoHeapImplBase::isNowFreeable): (bmalloc::IsoHeapImplBase::isNoLongerFreeable): (bmalloc::IsoHeapImpl<Config>::allocateFromShared): (bmalloc::IsoHeapImpl<Config>::freeableMemory): Deleted. (bmalloc::IsoHeapImpl<Config>::footprint): Deleted. (bmalloc::IsoHeapImpl<Config>::didCommit): Deleted. (bmalloc::IsoHeapImpl<Config>::didDecommit): Deleted. (bmalloc::IsoHeapImpl<Config>::isNowFreeable): Deleted. (bmalloc::IsoHeapImpl<Config>::isNoLongerFreeable): Deleted. * bmalloc/IsoPage.h: (bmalloc::IsoPageBase::IsoPageBase): * bmalloc/IsoPageInlines.h: (bmalloc::IsoPage<Config>::IsoPage): (bmalloc::IsoPage<Config>::free): (bmalloc::IsoPage<Config>::startAllocating): (bmalloc::IsoPage<Config>::stopAllocating): (bmalloc::IsoPage<Config>::forEachLiveObject): * bmalloc/IsoSharedHeap.h: (bmalloc::IsoSharedHeap::IsoSharedHeap): * bmalloc/IsoSharedHeapInlines.h: (bmalloc::IsoSharedHeap::allocateNew): (bmalloc::IsoSharedHeap::allocateSlow): * bmalloc/IsoSharedPage.h: * bmalloc/IsoSharedPageInlines.h: (bmalloc::IsoSharedPage::free): (bmalloc::IsoSharedPage::startAllocating): (bmalloc::IsoSharedPage::stopAllocating): * bmalloc/IsoTLS.h: * bmalloc/IsoTLSAllocatorEntry.h: * bmalloc/IsoTLSAllocatorEntryInlines.h: (bmalloc::IsoTLSAllocatorEntry<Config>::scavenge): * bmalloc/IsoTLSDeallocatorEntry.h: * bmalloc/IsoTLSDeallocatorEntryInlines.h: (bmalloc::IsoTLSDeallocatorEntry<Config>::scavenge): * bmalloc/IsoTLSEntry.cpp: (bmalloc::IsoTLSEntry::IsoTLSEntry): * bmalloc/IsoTLSEntry.h: * bmalloc/IsoTLSEntryInlines.h: (bmalloc::DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry): (bmalloc::DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry): Deleted. (bmalloc::DefaultIsoTLSEntry<EntryType>::scavenge): Deleted. * bmalloc/IsoTLSInlines.h: (bmalloc::IsoTLS::scavenge): (bmalloc::IsoTLS::allocateImpl): (bmalloc::IsoTLS::allocateFast): (bmalloc::IsoTLS::allocateSlow): * bmalloc/IsoTLSLayout.cpp: (bmalloc::IsoTLSLayout::add): * bmalloc/Packed.h: Added. (bmalloc::Packed::Packed): (bmalloc::Packed::get const): (bmalloc::Packed::set): (bmalloc::Packed::operator=): (bmalloc::Packed::exchange): (bmalloc::Packed::swap): (bmalloc::alignof): (bmalloc::PackedPtrTraits::exchange): (bmalloc::PackedPtrTraits::swap): (bmalloc::PackedPtrTraits::unwrap): * bmalloc/Scavenger.cpp: (bmalloc::Scavenger::Scavenger): * bmalloc/Scavenger.h: * bmalloc/VMHeap.cpp: (bmalloc::VMHeap::VMHeap): * bmalloc/VMHeap.h: * bmalloc/Zone.cpp: (bmalloc::Zone::Zone): * bmalloc/Zone.h: Tools: * TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp: (assertHasObjects): (assertHasOnlyObjects): (assertClean): (TEST): Canonical link: https://commits.webkit.org/219455@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@254708 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-01-16 22:05:00 +00:00
/*
Define MacroAssemblerARM64E::numberOfPACBits based on OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH). https://bugs.webkit.org/show_bug.cgi?id=227147 rdar://78785309 Reviewed by Saam Barati. Source/bmalloc: For OS(DARWIN), define BOS_EFFECTIVE_ADDRESS_WIDTH in terms of MACH_VM_MAX_ADDRESS, which is provided by the SDK. This ensures that it is correct for each target OS(DARWIN) platform. * bmalloc/Algorithm.h: (bmalloc::clzConstexpr): (bmalloc::getMSBSetConstexpr): * bmalloc/BPlatform.h: * bmalloc/Gigacage.h: * bmalloc/ObjectTypeTable.h: * bmalloc/Packed.h: Source/JavaScriptCore: * assembler/MacroAssemblerARM64E.h: * bytecode/CodeOrigin.h: * runtime/JSString.h: * runtime/OptionsList.h: Source/WTF: For OS(DARWIN), define OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH) in terms of MACH_VM_MAX_ADDRESS, which is provided by the SDK. This ensures that it is correct for each target OS(DARWIN) platform. Also update an assertion in WTFAssertions.cpp to verify that address bits are less than 48. The purpose of this assertion is to ensure that our 64-bit NaN boxing encoding for JSValues will work. Hence, we should use the encoding limit for pointers of 48 bits. It no longer makes sense to assert based on OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH), because OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH) is defined in terms of MACH_VM_MAX_ADDRESS. * wtf/CagedPtr.h: * wtf/CompactPointerTuple.h: * wtf/PlatformOS.h: * wtf/WTFAssertions.cpp: * wtf/threads/Signals.cpp: Tools: * TestWebKitAPI/Tests/WTF/Packed.cpp: Canonical link: https://commits.webkit.org/238948@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@279028 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2021-06-18 01:44:42 +00:00
* Copyright (C) 2019-2021 Apple Inc. All rights reserved.
[bmalloc] IsoHeap's initial setup should be small https://bugs.webkit.org/show_bug.cgi?id=206214 Reviewed by Michael Saboff. Source/bmalloc: Keep IsoHeap related data structures small by using Packed technique. We start using IsoHeap for many classes, then it is important that we keep metadata for IsoHeap small because these data persistently exists. 1. We pass IsoHeapImpl<> as a parameter instead of holding it unnecessarily. 2. We make some of pointers Packed so that we can keep sizeof(IsoHeapImpl<Config>) small. 3. One of the drawback of PackedPtr is that loading and storing are not atomic. And we pass `const std::lock_guard<Mutex>&` to functions if functions need to be called with lock so that we ensure that PackedPtr are accessed only when lock is held correctly. * CMakeLists.txt: * bmalloc.xcodeproj/project.pbxproj: * bmalloc/Algorithm.h: (bmalloc::ctzConstexpr): (bmalloc::getLSBSetNonZeroConstexpr): * bmalloc/BPlatform.h: * bmalloc/DebugHeap.cpp: (bmalloc::DebugHeap::DebugHeap): * bmalloc/DebugHeap.h: * bmalloc/DeferredTrigger.h: * bmalloc/DeferredTriggerInlines.h: (bmalloc::DeferredTrigger<trigger>::didBecome): (bmalloc::DeferredTrigger<trigger>::handleDeferral): * bmalloc/Environment.cpp: (bmalloc::Environment::Environment): * bmalloc/Environment.h: * bmalloc/Gigacage.cpp: (bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks): * bmalloc/Heap.cpp: (bmalloc::Heap::freeableMemory): (bmalloc::Heap::markAllLargeAsEligibile): (bmalloc::Heap::decommitLargeRange): (bmalloc::Heap::scavenge): (bmalloc::Heap::scavengeToHighWatermark): * bmalloc/Heap.h: * bmalloc/HeapConstants.cpp: (bmalloc::HeapConstants::HeapConstants): * bmalloc/HeapConstants.h: * bmalloc/IsoAllocator.h: * bmalloc/IsoAllocatorInlines.h: (bmalloc::IsoAllocator<Config>::IsoAllocator): (bmalloc::IsoAllocator<Config>::allocate): (bmalloc::IsoAllocator<Config>::allocateSlow): (bmalloc::IsoAllocator<Config>::scavenge): * bmalloc/IsoDeallocatorInlines.h: (bmalloc::IsoDeallocator<Config>::scavenge): * bmalloc/IsoDirectory.h: * bmalloc/IsoDirectoryInlines.h: (bmalloc::passedNumPages>::IsoDirectory): (bmalloc::passedNumPages>::takeFirstEligible): (bmalloc::passedNumPages>::didBecome): (bmalloc::passedNumPages>::didDecommit): (bmalloc::passedNumPages>::scavengePage): (bmalloc::passedNumPages>::scavenge): (bmalloc::passedNumPages>::scavengeToHighWatermark): (bmalloc::passedNumPages>::forEachCommittedPage): * bmalloc/IsoHeapImpl.cpp: (bmalloc::IsoHeapImplBase::IsoHeapImplBase): * bmalloc/IsoHeapImpl.h: * bmalloc/IsoHeapImplInlines.h: (bmalloc::IsoHeapImpl<Config>::IsoHeapImpl): (bmalloc::IsoHeapImpl<Config>::takeFirstEligible): (bmalloc::IsoHeapImpl<Config>::didBecomeEligibleOrDecommited): (bmalloc::IsoHeapImpl<Config>::scavenge): (bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark): (bmalloc::IsoHeapImplBase::freeableMemory): (bmalloc::IsoHeapImpl<Config>::numLiveObjects): (bmalloc::IsoHeapImpl<Config>::numCommittedPages): (bmalloc::IsoHeapImpl<Config>::forEachDirectory): (bmalloc::IsoHeapImpl<Config>::forEachCommittedPage): (bmalloc::IsoHeapImpl<Config>::forEachLiveObject): (bmalloc::IsoHeapImplBase::footprint): (bmalloc::IsoHeapImplBase::didCommit): (bmalloc::IsoHeapImplBase::didDecommit): (bmalloc::IsoHeapImplBase::isNowFreeable): (bmalloc::IsoHeapImplBase::isNoLongerFreeable): (bmalloc::IsoHeapImpl<Config>::allocateFromShared): (bmalloc::IsoHeapImpl<Config>::freeableMemory): Deleted. (bmalloc::IsoHeapImpl<Config>::footprint): Deleted. (bmalloc::IsoHeapImpl<Config>::didCommit): Deleted. (bmalloc::IsoHeapImpl<Config>::didDecommit): Deleted. (bmalloc::IsoHeapImpl<Config>::isNowFreeable): Deleted. (bmalloc::IsoHeapImpl<Config>::isNoLongerFreeable): Deleted. * bmalloc/IsoPage.h: (bmalloc::IsoPageBase::IsoPageBase): * bmalloc/IsoPageInlines.h: (bmalloc::IsoPage<Config>::IsoPage): (bmalloc::IsoPage<Config>::free): (bmalloc::IsoPage<Config>::startAllocating): (bmalloc::IsoPage<Config>::stopAllocating): (bmalloc::IsoPage<Config>::forEachLiveObject): * bmalloc/IsoSharedHeap.h: (bmalloc::IsoSharedHeap::IsoSharedHeap): * bmalloc/IsoSharedHeapInlines.h: (bmalloc::IsoSharedHeap::allocateNew): (bmalloc::IsoSharedHeap::allocateSlow): * bmalloc/IsoSharedPage.h: * bmalloc/IsoSharedPageInlines.h: (bmalloc::IsoSharedPage::free): (bmalloc::IsoSharedPage::startAllocating): (bmalloc::IsoSharedPage::stopAllocating): * bmalloc/IsoTLS.h: * bmalloc/IsoTLSAllocatorEntry.h: * bmalloc/IsoTLSAllocatorEntryInlines.h: (bmalloc::IsoTLSAllocatorEntry<Config>::scavenge): * bmalloc/IsoTLSDeallocatorEntry.h: * bmalloc/IsoTLSDeallocatorEntryInlines.h: (bmalloc::IsoTLSDeallocatorEntry<Config>::scavenge): * bmalloc/IsoTLSEntry.cpp: (bmalloc::IsoTLSEntry::IsoTLSEntry): * bmalloc/IsoTLSEntry.h: * bmalloc/IsoTLSEntryInlines.h: (bmalloc::DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry): (bmalloc::DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry): Deleted. (bmalloc::DefaultIsoTLSEntry<EntryType>::scavenge): Deleted. * bmalloc/IsoTLSInlines.h: (bmalloc::IsoTLS::scavenge): (bmalloc::IsoTLS::allocateImpl): (bmalloc::IsoTLS::allocateFast): (bmalloc::IsoTLS::allocateSlow): * bmalloc/IsoTLSLayout.cpp: (bmalloc::IsoTLSLayout::add): * bmalloc/Packed.h: Added. (bmalloc::Packed::Packed): (bmalloc::Packed::get const): (bmalloc::Packed::set): (bmalloc::Packed::operator=): (bmalloc::Packed::exchange): (bmalloc::Packed::swap): (bmalloc::alignof): (bmalloc::PackedPtrTraits::exchange): (bmalloc::PackedPtrTraits::swap): (bmalloc::PackedPtrTraits::unwrap): * bmalloc/Scavenger.cpp: (bmalloc::Scavenger::Scavenger): * bmalloc/Scavenger.h: * bmalloc/VMHeap.cpp: (bmalloc::VMHeap::VMHeap): * bmalloc/VMHeap.h: * bmalloc/Zone.cpp: (bmalloc::Zone::Zone): * bmalloc/Zone.h: Tools: * TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp: (assertHasObjects): (assertHasOnlyObjects): (assertClean): (TEST): Canonical link: https://commits.webkit.org/219455@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@254708 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-01-16 22:05:00 +00:00
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include "Algorithm.h"
#include "StdLibExtras.h"
#include <array>
Define MacroAssemblerARM64E::numberOfPACBits based on OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH). https://bugs.webkit.org/show_bug.cgi?id=227147 rdar://78785309 Reviewed by Saam Barati. Source/bmalloc: For OS(DARWIN), define BOS_EFFECTIVE_ADDRESS_WIDTH in terms of MACH_VM_MAX_ADDRESS, which is provided by the SDK. This ensures that it is correct for each target OS(DARWIN) platform. * bmalloc/Algorithm.h: (bmalloc::clzConstexpr): (bmalloc::getMSBSetConstexpr): * bmalloc/BPlatform.h: * bmalloc/Gigacage.h: * bmalloc/ObjectTypeTable.h: * bmalloc/Packed.h: Source/JavaScriptCore: * assembler/MacroAssemblerARM64E.h: * bytecode/CodeOrigin.h: * runtime/JSString.h: * runtime/OptionsList.h: Source/WTF: For OS(DARWIN), define OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH) in terms of MACH_VM_MAX_ADDRESS, which is provided by the SDK. This ensures that it is correct for each target OS(DARWIN) platform. Also update an assertion in WTFAssertions.cpp to verify that address bits are less than 48. The purpose of this assertion is to ensure that our 64-bit NaN boxing encoding for JSValues will work. Hence, we should use the encoding limit for pointers of 48 bits. It no longer makes sense to assert based on OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH), because OS_CONSTANT(EFFECTIVE_ADDRESS_WIDTH) is defined in terms of MACH_VM_MAX_ADDRESS. * wtf/CagedPtr.h: * wtf/CompactPointerTuple.h: * wtf/PlatformOS.h: * wtf/WTFAssertions.cpp: * wtf/threads/Signals.cpp: Tools: * TestWebKitAPI/Tests/WTF/Packed.cpp: Canonical link: https://commits.webkit.org/238948@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@279028 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2021-06-18 01:44:42 +00:00
#if BOS(DARWIN)
#include <mach/vm_param.h>
#endif
[bmalloc] IsoHeap's initial setup should be small https://bugs.webkit.org/show_bug.cgi?id=206214 Reviewed by Michael Saboff. Source/bmalloc: Keep IsoHeap related data structures small by using Packed technique. We start using IsoHeap for many classes, then it is important that we keep metadata for IsoHeap small because these data persistently exists. 1. We pass IsoHeapImpl<> as a parameter instead of holding it unnecessarily. 2. We make some of pointers Packed so that we can keep sizeof(IsoHeapImpl<Config>) small. 3. One of the drawback of PackedPtr is that loading and storing are not atomic. And we pass `const std::lock_guard<Mutex>&` to functions if functions need to be called with lock so that we ensure that PackedPtr are accessed only when lock is held correctly. * CMakeLists.txt: * bmalloc.xcodeproj/project.pbxproj: * bmalloc/Algorithm.h: (bmalloc::ctzConstexpr): (bmalloc::getLSBSetNonZeroConstexpr): * bmalloc/BPlatform.h: * bmalloc/DebugHeap.cpp: (bmalloc::DebugHeap::DebugHeap): * bmalloc/DebugHeap.h: * bmalloc/DeferredTrigger.h: * bmalloc/DeferredTriggerInlines.h: (bmalloc::DeferredTrigger<trigger>::didBecome): (bmalloc::DeferredTrigger<trigger>::handleDeferral): * bmalloc/Environment.cpp: (bmalloc::Environment::Environment): * bmalloc/Environment.h: * bmalloc/Gigacage.cpp: (bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks): * bmalloc/Heap.cpp: (bmalloc::Heap::freeableMemory): (bmalloc::Heap::markAllLargeAsEligibile): (bmalloc::Heap::decommitLargeRange): (bmalloc::Heap::scavenge): (bmalloc::Heap::scavengeToHighWatermark): * bmalloc/Heap.h: * bmalloc/HeapConstants.cpp: (bmalloc::HeapConstants::HeapConstants): * bmalloc/HeapConstants.h: * bmalloc/IsoAllocator.h: * bmalloc/IsoAllocatorInlines.h: (bmalloc::IsoAllocator<Config>::IsoAllocator): (bmalloc::IsoAllocator<Config>::allocate): (bmalloc::IsoAllocator<Config>::allocateSlow): (bmalloc::IsoAllocator<Config>::scavenge): * bmalloc/IsoDeallocatorInlines.h: (bmalloc::IsoDeallocator<Config>::scavenge): * bmalloc/IsoDirectory.h: * bmalloc/IsoDirectoryInlines.h: (bmalloc::passedNumPages>::IsoDirectory): (bmalloc::passedNumPages>::takeFirstEligible): (bmalloc::passedNumPages>::didBecome): (bmalloc::passedNumPages>::didDecommit): (bmalloc::passedNumPages>::scavengePage): (bmalloc::passedNumPages>::scavenge): (bmalloc::passedNumPages>::scavengeToHighWatermark): (bmalloc::passedNumPages>::forEachCommittedPage): * bmalloc/IsoHeapImpl.cpp: (bmalloc::IsoHeapImplBase::IsoHeapImplBase): * bmalloc/IsoHeapImpl.h: * bmalloc/IsoHeapImplInlines.h: (bmalloc::IsoHeapImpl<Config>::IsoHeapImpl): (bmalloc::IsoHeapImpl<Config>::takeFirstEligible): (bmalloc::IsoHeapImpl<Config>::didBecomeEligibleOrDecommited): (bmalloc::IsoHeapImpl<Config>::scavenge): (bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark): (bmalloc::IsoHeapImplBase::freeableMemory): (bmalloc::IsoHeapImpl<Config>::numLiveObjects): (bmalloc::IsoHeapImpl<Config>::numCommittedPages): (bmalloc::IsoHeapImpl<Config>::forEachDirectory): (bmalloc::IsoHeapImpl<Config>::forEachCommittedPage): (bmalloc::IsoHeapImpl<Config>::forEachLiveObject): (bmalloc::IsoHeapImplBase::footprint): (bmalloc::IsoHeapImplBase::didCommit): (bmalloc::IsoHeapImplBase::didDecommit): (bmalloc::IsoHeapImplBase::isNowFreeable): (bmalloc::IsoHeapImplBase::isNoLongerFreeable): (bmalloc::IsoHeapImpl<Config>::allocateFromShared): (bmalloc::IsoHeapImpl<Config>::freeableMemory): Deleted. (bmalloc::IsoHeapImpl<Config>::footprint): Deleted. (bmalloc::IsoHeapImpl<Config>::didCommit): Deleted. (bmalloc::IsoHeapImpl<Config>::didDecommit): Deleted. (bmalloc::IsoHeapImpl<Config>::isNowFreeable): Deleted. (bmalloc::IsoHeapImpl<Config>::isNoLongerFreeable): Deleted. * bmalloc/IsoPage.h: (bmalloc::IsoPageBase::IsoPageBase): * bmalloc/IsoPageInlines.h: (bmalloc::IsoPage<Config>::IsoPage): (bmalloc::IsoPage<Config>::free): (bmalloc::IsoPage<Config>::startAllocating): (bmalloc::IsoPage<Config>::stopAllocating): (bmalloc::IsoPage<Config>::forEachLiveObject): * bmalloc/IsoSharedHeap.h: (bmalloc::IsoSharedHeap::IsoSharedHeap): * bmalloc/IsoSharedHeapInlines.h: (bmalloc::IsoSharedHeap::allocateNew): (bmalloc::IsoSharedHeap::allocateSlow): * bmalloc/IsoSharedPage.h: * bmalloc/IsoSharedPageInlines.h: (bmalloc::IsoSharedPage::free): (bmalloc::IsoSharedPage::startAllocating): (bmalloc::IsoSharedPage::stopAllocating): * bmalloc/IsoTLS.h: * bmalloc/IsoTLSAllocatorEntry.h: * bmalloc/IsoTLSAllocatorEntryInlines.h: (bmalloc::IsoTLSAllocatorEntry<Config>::scavenge): * bmalloc/IsoTLSDeallocatorEntry.h: * bmalloc/IsoTLSDeallocatorEntryInlines.h: (bmalloc::IsoTLSDeallocatorEntry<Config>::scavenge): * bmalloc/IsoTLSEntry.cpp: (bmalloc::IsoTLSEntry::IsoTLSEntry): * bmalloc/IsoTLSEntry.h: * bmalloc/IsoTLSEntryInlines.h: (bmalloc::DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry): (bmalloc::DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry): Deleted. (bmalloc::DefaultIsoTLSEntry<EntryType>::scavenge): Deleted. * bmalloc/IsoTLSInlines.h: (bmalloc::IsoTLS::scavenge): (bmalloc::IsoTLS::allocateImpl): (bmalloc::IsoTLS::allocateFast): (bmalloc::IsoTLS::allocateSlow): * bmalloc/IsoTLSLayout.cpp: (bmalloc::IsoTLSLayout::add): * bmalloc/Packed.h: Added. (bmalloc::Packed::Packed): (bmalloc::Packed::get const): (bmalloc::Packed::set): (bmalloc::Packed::operator=): (bmalloc::Packed::exchange): (bmalloc::Packed::swap): (bmalloc::alignof): (bmalloc::PackedPtrTraits::exchange): (bmalloc::PackedPtrTraits::swap): (bmalloc::PackedPtrTraits::unwrap): * bmalloc/Scavenger.cpp: (bmalloc::Scavenger::Scavenger): * bmalloc/Scavenger.h: * bmalloc/VMHeap.cpp: (bmalloc::VMHeap::VMHeap): * bmalloc/VMHeap.h: * bmalloc/Zone.cpp: (bmalloc::Zone::Zone): * bmalloc/Zone.h: Tools: * TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp: (assertHasObjects): (assertHasOnlyObjects): (assertClean): (TEST): Canonical link: https://commits.webkit.org/219455@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@254708 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-01-16 22:05:00 +00:00
namespace bmalloc {
template<typename T>
class Packed {
public:
static_assert(std::is_trivial<T>::value);
static constexpr bool isPackedType = true;
Packed()
: Packed(T { })
{
}
Packed(const T& value)
{
memcpy(m_storage.data(), &value, sizeof(T));
}
T get() const
{
T value { };
memcpy(&value, m_storage.data(), sizeof(T));
return value;
}
void set(const T& value)
{
memcpy(m_storage.data(), &value, sizeof(T));
}
Packed<T>& operator=(const T& value)
{
set(value);
return *this;
}
template<class U>
T exchange(U&& newValue)
{
T oldValue = get();
set(std::forward<U>(newValue));
return oldValue;
}
void swap(Packed& other)
{
m_storage.swap(other.m_storage);
}
template<typename Other, typename = std::enable_if_t<Other::isPackedType>>
void swap(Other& other)
{
T t1 = get();
T t2 = other.get();
set(t2);
other.set(t1);
}
void swap(T& t2)
{
T t1 = get();
std::swap(t1, t2);
set(t1);
}
private:
std::array<uint8_t, sizeof(T)> m_storage;
};
// PackedAlignedPtr can take alignment parameter too. PackedAlignedPtr only uses this alignment information if it is profitable: we use
// alignment information only when we can reduce the size of the storage.
[bmalloc] IsoHeap's initial setup should be small https://bugs.webkit.org/show_bug.cgi?id=206214 Reviewed by Michael Saboff. Source/bmalloc: Keep IsoHeap related data structures small by using Packed technique. We start using IsoHeap for many classes, then it is important that we keep metadata for IsoHeap small because these data persistently exists. 1. We pass IsoHeapImpl<> as a parameter instead of holding it unnecessarily. 2. We make some of pointers Packed so that we can keep sizeof(IsoHeapImpl<Config>) small. 3. One of the drawback of PackedPtr is that loading and storing are not atomic. And we pass `const std::lock_guard<Mutex>&` to functions if functions need to be called with lock so that we ensure that PackedPtr are accessed only when lock is held correctly. * CMakeLists.txt: * bmalloc.xcodeproj/project.pbxproj: * bmalloc/Algorithm.h: (bmalloc::ctzConstexpr): (bmalloc::getLSBSetNonZeroConstexpr): * bmalloc/BPlatform.h: * bmalloc/DebugHeap.cpp: (bmalloc::DebugHeap::DebugHeap): * bmalloc/DebugHeap.h: * bmalloc/DeferredTrigger.h: * bmalloc/DeferredTriggerInlines.h: (bmalloc::DeferredTrigger<trigger>::didBecome): (bmalloc::DeferredTrigger<trigger>::handleDeferral): * bmalloc/Environment.cpp: (bmalloc::Environment::Environment): * bmalloc/Environment.h: * bmalloc/Gigacage.cpp: (bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks): * bmalloc/Heap.cpp: (bmalloc::Heap::freeableMemory): (bmalloc::Heap::markAllLargeAsEligibile): (bmalloc::Heap::decommitLargeRange): (bmalloc::Heap::scavenge): (bmalloc::Heap::scavengeToHighWatermark): * bmalloc/Heap.h: * bmalloc/HeapConstants.cpp: (bmalloc::HeapConstants::HeapConstants): * bmalloc/HeapConstants.h: * bmalloc/IsoAllocator.h: * bmalloc/IsoAllocatorInlines.h: (bmalloc::IsoAllocator<Config>::IsoAllocator): (bmalloc::IsoAllocator<Config>::allocate): (bmalloc::IsoAllocator<Config>::allocateSlow): (bmalloc::IsoAllocator<Config>::scavenge): * bmalloc/IsoDeallocatorInlines.h: (bmalloc::IsoDeallocator<Config>::scavenge): * bmalloc/IsoDirectory.h: * bmalloc/IsoDirectoryInlines.h: (bmalloc::passedNumPages>::IsoDirectory): (bmalloc::passedNumPages>::takeFirstEligible): (bmalloc::passedNumPages>::didBecome): (bmalloc::passedNumPages>::didDecommit): (bmalloc::passedNumPages>::scavengePage): (bmalloc::passedNumPages>::scavenge): (bmalloc::passedNumPages>::scavengeToHighWatermark): (bmalloc::passedNumPages>::forEachCommittedPage): * bmalloc/IsoHeapImpl.cpp: (bmalloc::IsoHeapImplBase::IsoHeapImplBase): * bmalloc/IsoHeapImpl.h: * bmalloc/IsoHeapImplInlines.h: (bmalloc::IsoHeapImpl<Config>::IsoHeapImpl): (bmalloc::IsoHeapImpl<Config>::takeFirstEligible): (bmalloc::IsoHeapImpl<Config>::didBecomeEligibleOrDecommited): (bmalloc::IsoHeapImpl<Config>::scavenge): (bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark): (bmalloc::IsoHeapImplBase::freeableMemory): (bmalloc::IsoHeapImpl<Config>::numLiveObjects): (bmalloc::IsoHeapImpl<Config>::numCommittedPages): (bmalloc::IsoHeapImpl<Config>::forEachDirectory): (bmalloc::IsoHeapImpl<Config>::forEachCommittedPage): (bmalloc::IsoHeapImpl<Config>::forEachLiveObject): (bmalloc::IsoHeapImplBase::footprint): (bmalloc::IsoHeapImplBase::didCommit): (bmalloc::IsoHeapImplBase::didDecommit): (bmalloc::IsoHeapImplBase::isNowFreeable): (bmalloc::IsoHeapImplBase::isNoLongerFreeable): (bmalloc::IsoHeapImpl<Config>::allocateFromShared): (bmalloc::IsoHeapImpl<Config>::freeableMemory): Deleted. (bmalloc::IsoHeapImpl<Config>::footprint): Deleted. (bmalloc::IsoHeapImpl<Config>::didCommit): Deleted. (bmalloc::IsoHeapImpl<Config>::didDecommit): Deleted. (bmalloc::IsoHeapImpl<Config>::isNowFreeable): Deleted. (bmalloc::IsoHeapImpl<Config>::isNoLongerFreeable): Deleted. * bmalloc/IsoPage.h: (bmalloc::IsoPageBase::IsoPageBase): * bmalloc/IsoPageInlines.h: (bmalloc::IsoPage<Config>::IsoPage): (bmalloc::IsoPage<Config>::free): (bmalloc::IsoPage<Config>::startAllocating): (bmalloc::IsoPage<Config>::stopAllocating): (bmalloc::IsoPage<Config>::forEachLiveObject): * bmalloc/IsoSharedHeap.h: (bmalloc::IsoSharedHeap::IsoSharedHeap): * bmalloc/IsoSharedHeapInlines.h: (bmalloc::IsoSharedHeap::allocateNew): (bmalloc::IsoSharedHeap::allocateSlow): * bmalloc/IsoSharedPage.h: * bmalloc/IsoSharedPageInlines.h: (bmalloc::IsoSharedPage::free): (bmalloc::IsoSharedPage::startAllocating): (bmalloc::IsoSharedPage::stopAllocating): * bmalloc/IsoTLS.h: * bmalloc/IsoTLSAllocatorEntry.h: * bmalloc/IsoTLSAllocatorEntryInlines.h: (bmalloc::IsoTLSAllocatorEntry<Config>::scavenge): * bmalloc/IsoTLSDeallocatorEntry.h: * bmalloc/IsoTLSDeallocatorEntryInlines.h: (bmalloc::IsoTLSDeallocatorEntry<Config>::scavenge): * bmalloc/IsoTLSEntry.cpp: (bmalloc::IsoTLSEntry::IsoTLSEntry): * bmalloc/IsoTLSEntry.h: * bmalloc/IsoTLSEntryInlines.h: (bmalloc::DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry): (bmalloc::DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry): Deleted. (bmalloc::DefaultIsoTLSEntry<EntryType>::scavenge): Deleted. * bmalloc/IsoTLSInlines.h: (bmalloc::IsoTLS::scavenge): (bmalloc::IsoTLS::allocateImpl): (bmalloc::IsoTLS::allocateFast): (bmalloc::IsoTLS::allocateSlow): * bmalloc/IsoTLSLayout.cpp: (bmalloc::IsoTLSLayout::add): * bmalloc/Packed.h: Added. (bmalloc::Packed::Packed): (bmalloc::Packed::get const): (bmalloc::Packed::set): (bmalloc::Packed::operator=): (bmalloc::Packed::exchange): (bmalloc::Packed::swap): (bmalloc::alignof): (bmalloc::PackedPtrTraits::exchange): (bmalloc::PackedPtrTraits::swap): (bmalloc::PackedPtrTraits::unwrap): * bmalloc/Scavenger.cpp: (bmalloc::Scavenger::Scavenger): * bmalloc/Scavenger.h: * bmalloc/VMHeap.cpp: (bmalloc::VMHeap::VMHeap): * bmalloc/VMHeap.h: * bmalloc/Zone.cpp: (bmalloc::Zone::Zone): * bmalloc/Zone.h: Tools: * TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp: (assertHasObjects): (assertHasOnlyObjects): (assertClean): (TEST): Canonical link: https://commits.webkit.org/219455@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@254708 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2020-01-16 22:05:00 +00:00
template<typename T, size_t alignment = alignof(T)>
class PackedAlignedPtr {
public:
static_assert(isPowerOfTwo(alignment), "Alignment needs to be power-of-two");
static constexpr bool isPackedType = true;
static constexpr unsigned alignmentShiftSizeIfProfitable = getLSBSetNonZeroConstexpr(alignment);
static constexpr unsigned storageSizeWithoutAlignmentShift = roundUpToMultipleOf<8, uintptr_t>(BOS_EFFECTIVE_ADDRESS_WIDTH) / 8;
static constexpr unsigned storageSizeWithAlignmentShift = roundUpToMultipleOf<8, uintptr_t>(BOS_EFFECTIVE_ADDRESS_WIDTH - alignmentShiftSizeIfProfitable) / 8;
static constexpr bool isAlignmentShiftProfitable = storageSizeWithoutAlignmentShift > storageSizeWithAlignmentShift;
static constexpr unsigned alignmentShiftSize = isAlignmentShiftProfitable ? alignmentShiftSizeIfProfitable : 0;
static constexpr unsigned storageSize = storageSizeWithAlignmentShift;
constexpr PackedAlignedPtr()
: m_storage()
{
}
constexpr PackedAlignedPtr(std::nullptr_t)
: m_storage()
{
}
PackedAlignedPtr(T* value)
{
set(value);
}
T* get() const
{
// FIXME: PackedPtr<> can load memory with one mov by checking page boundary.
// https://bugs.webkit.org/show_bug.cgi?id=197754
uintptr_t value = 0;
#if BCPU(LITTLE_ENDIAN)
memcpy(&value, m_storage.data(), storageSize);
#else
memcpy(bitwise_cast<uint8_t*>(&value) + (sizeof(void*) - storageSize), m_storage.data(), storageSize);
#endif
if (isAlignmentShiftProfitable)
value <<= alignmentShiftSize;
return bitwise_cast<T*>(value);
}
void set(T* passedValue)
{
uintptr_t value = bitwise_cast<uintptr_t>(passedValue);
if (isAlignmentShiftProfitable)
value >>= alignmentShiftSize;
#if BCPU(LITTLE_ENDIAN)
memcpy(m_storage.data(), &value, storageSize);
#else
memcpy(m_storage.data(), bitwise_cast<uint8_t*>(&value) + (sizeof(void*) - storageSize), storageSize);
#endif
}
void clear()
{
set(nullptr);
}
T* operator->() const { return get(); }
T& operator*() const { return *get(); }
bool operator!() const { return !get(); }
// This conversion operator allows implicit conversion to bool but not to other integer types.
typedef T* (PackedAlignedPtr::*UnspecifiedBoolType);
operator UnspecifiedBoolType() const { return get() ? &PackedAlignedPtr::m_storage : nullptr; }
explicit operator bool() const { return get(); }
PackedAlignedPtr& operator=(T* value)
{
set(value);
return *this;
}
template<class U>
T* exchange(U&& newValue)
{
T* oldValue = get();
set(std::forward<U>(newValue));
return oldValue;
}
void swap(std::nullptr_t) { clear(); }
void swap(PackedAlignedPtr& other)
{
m_storage.swap(other.m_storage);
}
template<typename Other, typename = std::enable_if_t<Other::isPackedType>>
void swap(Other& other)
{
T* t1 = get();
T* t2 = other.get();
set(t2);
other.set(t1);
}
void swap(T* t2)
{
T* t1 = get();
std::swap(t1, t2);
set(t1);
}
private:
std::array<uint8_t, storageSize> m_storage;
};
template<typename T>
class Packed<T*> : public PackedAlignedPtr<T, 1> {
public:
using Base = PackedAlignedPtr<T, 1>;
using Base::Base;
};
template<typename T>
using PackedPtr = Packed<T*>;
template<typename T>
struct PackedPtrTraits {
template<typename U> using RebindTraits = PackedPtrTraits<U>;
using StorageType = PackedPtr<T>;
template<class U> static T* exchange(StorageType& ptr, U&& newValue) { return ptr.exchange(newValue); }
template<typename Other> static void swap(PackedPtr<T>& a, Other& b) { a.swap(b); }
static T* unwrap(const StorageType& ptr) { return ptr.get(); }
};
} // namespace bmalloc