// Copyright 2019 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/objects/backing-store.h"
#include <cstring>
#include <optional>
#include "src/base/bits.h"
#include "src/execution/isolate.h"
#include "src/handles/global-handles.h"
#include "src/logging/counters.h"
#include "src/objects/js-array-buffer-inl.h"
#include "src/objects/managed.h"
#include "src/sandbox/check.h"
#include "src/sandbox/sandbox.h"
#if V8_ENABLE_WEBASSEMBLY
#include "src/trap-handler/trap-handler.h"
#include "src/wasm/wasm-constants.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-limits.h"
#include "src/wasm/wasm-module.h"
#include "src/wasm/wasm-objects-inl.h"
#endif // V8_ENABLE_WEBASSEMBLY
#define TRACE_BS(...) \
do { \
if (v8_flags.trace_backing_store) PrintF(__VA_ARGS__); \
} while (false)
namespace v8::internal {
namespace {
#if V8_ENABLE_WEBASSEMBLY && V8_TARGET_ARCH_64_BIT
constexpr size_t kFullGuardSize32 = uint64_t{8} * GB;
#endif
std::atomic<uint32_t> next_backing_store_id_{1};
// Allocation results are reported to UMA
//
// See wasm_memory_allocation_result in counters-definitions.h
enum class AllocationStatus {
kSuccess, // Succeeded on the first try
kSuccessAfterRetry, // Succeeded after garbage collection
kAddressSpaceLimitReachedFailure, // Failed because Wasm is at its address
// space limit
kOtherFailure // Failed for an unknown reason
};
base::AddressRegion GetReservedRegion(HasGuardRegions has_guard_regions,
WasmMemoryFlag wasm_memory,
void* buffer_start,
size_t byte_capacity) {
return base::AddressRegion(
reinterpret_cast<Address>(buffer_start),
BackingStore::GetWasmReservationSize(has_guard_regions, byte_capacity,
wasm_memory));
}
void RecordStatus(Isolate* isolate, AllocationStatus status) {
isolate->counters()->wasm_memory_allocation_result()->AddSample(
static_cast<int>(status));
}
} // namespace
size_t BackingStore::GetWasmReservationSize(HasGuardRegions has_guard_regions,
size_t byte_capacity,
WasmMemoryFlag wasm_memory) {
#if V8_TARGET_ARCH_64_BIT && V8_ENABLE_WEBASSEMBLY
const bool is_wasm_memory64 = wasm_memory == WasmMemoryFlag::kWasmMemory64;
DCHECK_IMPLIES(is_wasm_memory64 && has_guard_regions,
v8_flags.wasm_memory64_trap_handling);
if (has_guard_regions) {
if (is_wasm_memory64) {
DCHECK_LE(byte_capacity, wasm::kMaxMemory64Size);
#if V8_TARGET_ARCH_ARM64
// Reserving an extra guard page simplifies bounds checking in the common
// case.
return wasm::kMaxMemory64Size + AllocatePageSize();
#else
return wasm::kMaxMemory64Size;
#endif // V8_TARGET_ARCH_ARM64
} else {
static_assert(kFullGuardSize32 >= size_t{4} * GB);
DCHECK_LE(byte_capacity, size_t{4} * GB);
return kFullGuardSize32;
}
}
#else
DCHECK(!has_guard_regions);
#endif
return byte_capacity;
}
// The backing store for a Wasm shared memory remembers all the isolates
// with which it has been shared.
struct SharedWasmMemoryData {
std::vector<Isolate*> isolates_;
};
BackingStore::BackingStore(void* buffer_start, size_t byte_length,
size_t max_byte_length, size_t byte_capacity,
SharedFlag shared, ResizableFlag resizable,
ImmutableFlag immutable, WasmMemoryFlag wasm_memory,
HasGuardRegions has_guard_regions,
CustomDeleter custom_deleter,
EmptyDeleter empty_deleter)
: buffer_start_(buffer_start),
byte_length_(byte_length),
max_byte_length_(max_byte_length),
byte_capacity_(byte_capacity),
id_(next_backing_store_id_.fetch_add(1)) {
const bool is_wasm_memory = wasm_memory != WasmMemoryFlag::kNotWasm;
const bool is_wasm_memory64 = wasm_memory == WasmMemoryFlag::kWasmMemory64;
// TODO(v8:11111): RAB / GSAB - Wasm integration.
DCHECK_IMPLIES(is_wasm_memory64, is_wasm_memory);
DCHECK_IMPLIES(has_guard_regions, is_wasm_memory);
DCHECK_IMPLIES(is_wasm_memory, !resizable);
DCHECK_IMPLIES(resizable, !custom_deleter);
DCHECK_IMPLIES(!is_wasm_memory && !resizable,
byte_length_ == max_byte_length_);
DCHECK_GE(max_byte_length, byte_length);
DCHECK_GE(byte_capacity, max_byte_length);
// TODO(1445003): Demote to a DCHECK once we found the issue.
// Wasm memory should never be empty (== zero capacity). Otherwise
// {JSArrayBuffer::Attach} would replace it by the {EmptyBackingStore} and we
// loose information.
// This is particularly important for shared Wasm memory.
CHECK_IMPLIES(is_wasm_memory, byte_capacity != 0);
base::EnumSet<Flag, uint16_t> flags;
if (shared) flags.Add(kIsShared);
if (resizable) flags.Add(kIsResizableByJs);
if (immutable) flags.Add(kIsImmutable);
if (is_wasm_memory) flags.Add(kIsWasmMemory);
if (is_wasm_memory64) flags.Add(kIsWasmMemory64);
if (has_guard_regions) flags.Add(kHasGuardRegions);
if (custom_deleter) flags.Add(kCustomDeleter);
if (empty_deleter) flags.Add(kEmptyDeleter);
flags_.store(flags, std::memory_order_relaxed);
}
BackingStore::~BackingStore() {
GlobalBackingStoreRegistry::Unregister(this);
struct ClearSharedAllocator {
BackingStore* const bs;
~ClearSharedAllocator() {
if (!bs->holds_shared_ptr_to_allocator()) return;
bs->type_specific_data_.v8_api_array_buffer_allocator_shared
.std::shared_ptr<v8::ArrayBuffer::Allocator>::~shared_ptr();
}
} clear_shared_allocator{this};
if (buffer_start_ == nullptr) return;
auto FreeResizableMemory = [this] {
DCHECK(!custom_deleter());
DCHECK(is_resizable_by_js() || is_wasm_memory());
auto region = GetReservedRegion(has_guard_regions(), wasm_memory_flag(),
buffer_start_, byte_capacity_);
#ifdef V8_ENABLE_SANDBOX
if (!region.is_empty() && !page_allocator_.expired()) {
auto page_allocator = page_allocator_.lock();
FreePages(page_allocator.get(), reinterpret_cast<void*>(region.begin()),
region.size());
}
#else
if (!region.is_empty()) {
FreePages(GetPlatformPageAllocator(),
reinterpret_cast<void*>(region.begin()), region.size());
}
#endif
};
#if V8_ENABLE_WEBASSEMBLY
if (is_wasm_memory()) {
size_t reservation_size = GetWasmReservationSize(
has_guard_regions(), byte_capacity_, wasm_memory_flag());
TRACE_BS(
"BSw:free bs=%p mem=%p (length=%zu, capacity=%zu, reservation=%zu)\n",
this, buffer_start_, byte_length(), byte_capacity_, reservation_size);
if (has_guard_regions()) {
trap_handler::UnregisterCoveredMemory(
reinterpret_cast<uintptr_t>(buffer_start_), reservation_size);
}
if (is_shared()) {
// Deallocate the list of attached memory objects.
SharedWasmMemoryData* shared_data = get_shared_wasm_memory_data();
delete shared_data;
}
// Wasm memories are always allocated through the page allocator.
FreeResizableMemory();
return;
}
#endif // V8_ENABLE_WEBASSEMBLY
if (is_resizable_by_js()) {
FreeResizableMemory();
return;
}
if (custom_deleter()) {
TRACE_BS("BS:custom deleter bs=%p mem=%p (length=%zu, capacity=%zu)\n",
this, buffer_start_, byte_length(), byte_capacity_);
type_specific_data_.deleter.callback(buffer_start_, byte_length_,
type_specific_data_.deleter.data);
return;
}
// JSArrayBuffer backing store. Deallocate through the embedder's allocator.
auto allocator = get_v8_api_array_buffer_allocator();
TRACE_BS("BS:free bs=%p mem=%p (length=%zu, capacity=%zu)\n", this,
buffer_start_, byte_length(), byte_capacity_);
allocator->Free(buffer_start_, byte_length_);
}
// Allocate a backing store using the array buffer allocator from the embedder.
std::unique_ptr<BackingStore> BackingStore::Allocate(
Isolate* isolate, size_t byte_length, SharedFlag shared,
InitializedFlag initialized) {
void* buffer_start = nullptr;
auto allocator = isolate->array_buffer_allocator();
CHECK_NOT_NULL(allocator);
if (byte_length > allocator->MaxAllocationSize()) return {};
if (byte_length != 0) {
auto counters = isolate->counters();
int mb_length = static_cast<int>(byte_length / MB);
if (mb_length > 0) {
counters->array_buffer_big_allocations()->AddSample(mb_length);
}
auto allocate_buffer = [allocator, initialized](size_t byte_length) {
if (!initialized) {
return allocator->AllocateUninitialized(byte_length);
}
return allocator->Allocate(byte_length);
};
buffer_start = isolate->heap()->AllocateExternalBackingStore(
allocate_buffer, byte_length);
if (buffer_start == nullptr) {
// Allocation failed.
counters->array_buffer_new_size_failures()->AddSample(mb_length);
return {};
}
#ifdef V8_ENABLE_SANDBOX
// Check to catch use of a non-sandbox-compatible ArrayBufferAllocator.
CHECK_WITH_MSG(isolate->isolate_group()->sandbox()->Contains(buffer_start),
"When the V8 Sandbox is enabled, ArrayBuffer backing stores "
"must be allocated inside the sandbox address space. Please "
"use an appropriate ArrayBuffer::Allocator to allocate "
"these buffers, or disable the sandbox.");
#endif
}
auto result =
new BackingStore(buffer_start, // start
byte_length, // length
byte_length, // max length
byte_length, // capacity
shared, ResizableFlag{false}, ImmutableFlag{false},
WasmMemoryFlag::kNotWasm, HasGuardRegions{false},
CustomDeleter{false}, EmptyDeleter{false});
#ifdef V8_ENABLE_SANDBOX
result->set_page_allocator(
isolate->isolate_group()->GetBackingStorePageAllocator());
#endif
TRACE_BS("BS:alloc bs=%p mem=%p (length=%zu)\n", result,
result->buffer_start(), byte_length);
result->SetAllocatorFromIsolate(isolate);
return std::unique_ptr<BackingStore>(result);
}
void BackingStore::SetAllocatorFromIsolate(Isolate* isolate) {
if (auto allocator_shared = isolate->array_buffer_allocator_shared()) {
set_flag(kHoldsSharedPtrToAllocater);
new (&type_specific_data_.v8_api_array_buffer_allocator_shared)
std::shared_ptr<v8::ArrayBuffer::Allocator>(
std::move(allocator_shared));
} else {
type_specific_data_.v8_api_array_buffer_allocator =
isolate->array_buffer_allocator();
}
}
std::unique_ptr<BackingStore> BackingStore::TryAllocateAndPartiallyCommitMemory(
Isolate* isolate, size_t byte_length, size_t max_byte_length,
size_t page_size, size_t initial_pages, size_t maximum_pages,
WasmMemoryFlag wasm_memory, SharedFlag shared,
HasGuardRegions has_guard_regions) {
// Enforce engine limitation on the maximum number of pages.
if (maximum_pages > std::numeric_limits<size_t>::max() / page_size) {
return nullptr;
}
// Cannot reserve 0 pages on some OSes.
if (maximum_pages == 0) maximum_pages = 1;
TRACE_BS("BSw:try %zu pages, %zu max\n", initial_pages, maximum_pages);
#if V8_ENABLE_WEBASSEMBLY
bool is_wasm_memory = wasm_memory != WasmMemoryFlag::kNotWasm;
#else
CHECK_EQ(WasmMemoryFlag::kNotWasm, wasm_memory);
constexpr bool is_wasm_memory = false;
#endif // V8_ENABLE_WEBASSEMBLY
DCHECK_IMPLIES(has_guard_regions, is_wasm_memory);
// For accounting purposes, whether a GC was necessary.
bool did_retry = false;
auto gc_retry = [&](const auto& fn) {
if (fn()) return true;
// Collect garbage and retry.
did_retry = true;
return isolate->heap()->allocator()->RetryCustomAllocate(
fn, internal::AllocationType::kOld,
GarbageCollectionReason::kAllocationFailure);
};
size_t byte_capacity = maximum_pages * page_size;
size_t reservation_size =
GetWasmReservationSize(has_guard_regions, byte_capacity, wasm_memory);
//--------------------------------------------------------------------------
// Allocate pages (inaccessible by default).
//--------------------------------------------------------------------------
void* allocation_base = nullptr;
#ifdef V8_COMPRESS_POINTERS_IN_MULTIPLE_CAGES
#ifdef V8_ENABLE_SANDBOX
CHECK_WITH_MSG(isolate || Sandbox::current(),
#else
CHECK_WITH_MSG(isolate || IsolateGroup::current(),
#endif
"One must enter an v8::Isolate before allocating resizable "
"array backing stores");
#endif
#ifdef V8_ENABLE_SANDBOX
IsolateGroup* group =
isolate ? isolate->isolate_group() : IsolateGroup::current();
DCHECK(group);
std::shared_ptr<PageAllocator> page_allocator_shared_ptr =
group->GetBackingStorePageAllocator().lock();
PageAllocator* page_allocator = page_allocator_shared_ptr.get();
#else
PageAllocator* page_allocator = GetPlatformPageAllocator();
#endif
auto allocate_pages = [&] {
allocation_base = AllocatePages(page_allocator, reservation_size, page_size,
PageAllocator::kNoAccess);
return allocation_base != nullptr;
};
if (!gc_retry(allocate_pages)) {
// Page allocator could not reserve enough pages.
if (isolate != nullptr) {
RecordStatus(isolate, AllocationStatus::kOtherFailure);
}
TRACE_BS("BSw:try failed to allocate pages\n");
return {};
}
uint8_t* buffer_start = reinterpret_cast<uint8_t*>(allocation_base);
//--------------------------------------------------------------------------
// Commit the initial pages (allow read/write).
//--------------------------------------------------------------------------
size_t committed_byte_length = initial_pages * page_size;
auto commit_memory = [&] {
return committed_byte_length == 0 ||
SetPermissions(page_allocator, buffer_start, committed_byte_length,
PageAllocator::kReadWrite);
};
if (!gc_retry(commit_memory)) {
TRACE_BS("BSw:try failed to set permissions (%p, %zu)\n", buffer_start,
committed_byte_length);
FreePages(page_allocator, allocation_base, reservation_size);
// SetPermissions put us over the process memory limit.
// We return an empty result so that the caller can throw an exception.
return {};
}
if (isolate != nullptr) {
RecordStatus(isolate, did_retry ? AllocationStatus::kSuccessAfterRetry
: AllocationStatus::kSuccess);
}
ResizableFlag resizable =
is_wasm_memory ? ResizableFlag{false} : ResizableFlag{true};
auto result = new BackingStore(
buffer_start, byte_length, max_byte_length, byte_capacity, shared,
resizable, ImmutableFlag{false}, wasm_memory, has_guard_regions,
CustomDeleter{false}, EmptyDeleter{false});
#ifdef V8_ENABLE_SANDBOX
if (page_allocator_shared_ptr) {
result->set_page_allocator(page_allocator_shared_ptr);
}
#endif
TRACE_BS(
"BSw:alloc bs=%p mem=%p (length=%zu, capacity=%zu, reservation=%zu)\n",
result, result->buffer_start(), byte_length, byte_capacity,
reservation_size);
return std::unique_ptr<BackingStore>(result);
}
#if V8_ENABLE_WEBASSEMBLY
// Allocate a backing store for a Wasm memory. Always use the page allocator
// and add guard regions.
std::unique_ptr<BackingStore> BackingStore::AllocateWasmMemory(
Isolate* isolate, size_t initial_pages, size_t maximum_pages,
WasmMemoryFlag wasm_memory, SharedFlag shared) {
// Wasm pages must be a multiple of the allocation page size.
DCHECK_EQ(0, wasm::kWasmPageSize % AllocatePageSize());
DCHECK_LE(initial_pages, maximum_pages);
DCHECK_LE(maximum_pages, wasm_memory == WasmMemoryFlag::kWasmMemory32
? wasm::kV8MaxWasmMemory32Pages
: wasm::kV8MaxWasmMemory64Pages);
DCHECK(wasm_memory == WasmMemoryFlag::kWasmMemory32 ||
wasm_memory == WasmMemoryFlag::kWasmMemory64);
bool is_wasm_memory64 = wasm_memory == WasmMemoryFlag::kWasmMemory64;
HasGuardRegions has_guard_regions{
trap_handler::IsTrapHandlerEnabled() &&
(wasm_memory == WasmMemoryFlag::kWasmMemory32 ||
(is_wasm_memory64 && v8_flags.wasm_memory64_trap_handling))};
auto TryAllocate = [isolate, initial_pages, wasm_memory, shared,
has_guard_regions](size_t maximum_pages) {
auto result = TryAllocateAndPartiallyCommitMemory(
isolate, initial_pages * wasm::kWasmPageSize,
maximum_pages * wasm::kWasmPageSize, wasm::kWasmPageSize, initial_pages,
maximum_pages, wasm_memory, shared, has_guard_regions);
if (result && shared) {
result->type_specific_data_.shared_wasm_memory_data =
new SharedWasmMemoryData();
}
return result;
};
auto backing_store = TryAllocate(maximum_pages);
if (!backing_store &&
!has_guard_regions && // With guard regions we always reserved a fixed
// number of pages.
maximum_pages - initial_pages >= 4) {
// Retry with smaller maximum pages at each retry.
auto delta = (maximum_pages - initial_pages) / 4;
size_t sizes[] = {maximum_pages - delta, maximum_pages - 2 * delta,
maximum_pages - 3 * delta, initial_pages};
for (size_t reduced_maximum_pages : sizes) {
backing_store = TryAllocate(reduced_maximum_pages);
if (backing_store) break;
}
}
if (backing_store && has_guard_regions) {
size_t reservation_size = GetWasmReservationSize(
backing_store->has_guard_regions(), backing_store->byte_capacity(),
backing_store->wasm_memory_flag());
if (!trap_handler::RegisterCoveredMemory(
reinterpret_cast<uintptr_t>(backing_store->buffer_start()),
reservation_size)) {
V8::FatalProcessOutOfMemory(
isolate, "Failed to register memory with the trap handler");
}
}
return backing_store;
}
std::unique_ptr<BackingStore> BackingStore::CopyWasmMemory(
Isolate* isolate, size_t new_pages, size_t max_pages,
WasmMemoryFlag wasm_memory) {
// Note that we could allocate uninitialized to save initialization cost here,
// but since Wasm memories are allocated by the page allocator, the zeroing
// cost is already built-in.
auto new_backing_store = BackingStore::AllocateWasmMemory(
isolate, new_pages, max_pages, wasm_memory, SharedFlag(is_shared()));
if (!new_backing_store ||
new_backing_store->has_guard_regions() != has_guard_regions()) {
return {};
}
if (byte_length_ > 0) {
// If the allocation was successful, then the new buffer must be at least
// as big as the old one.
DCHECK_GE(new_pages * wasm::kWasmPageSize, byte_length_);
memcpy(new_backing_store->buffer_start(), buffer_start_, byte_length_);
}
return new_backing_store;
}
// Try to grow the size of a wasm memory in place, without realloc + copy.
// Returns the previous number of pages on success.
std::optional<size_t> BackingStore::GrowWasmMemoryInPlace(Isolate* isolate,
size_t delta_pages,
size_t max_pages) {
// This function grows wasm memory by
// * changing the permissions of additional {delta_pages} pages to kReadWrite;
// * increment {byte_length_};
//
// As this code is executed concurrently, the following steps are executed:
// 1) Read the current value of {byte_length_};
// 2) Change the permission of all pages from {buffer_start_} to
// {byte_length_} + {delta_pages} * {page_size} to kReadWrite;
// * This operation may be executed racefully. The OS takes care of
// synchronization.
// 3) Try to update {byte_length_} with a compare_exchange;
// 4) Repeat 1) to 3) until the compare_exchange in 3) succeeds;
//
// The result of this function is the {byte_length_} before growing in pages.
// The result of this function appears like the result of an RMW-update on
// {byte_length_}, i.e. two concurrent calls to this function will result in
// different return values if {delta_pages} != 0.
//
// Invariants:
// * Permissions are always set incrementally, i.e. for any page {b} with
// kReadWrite permission, all pages between the first page {a} and page {b}
// also have kReadWrite permission.
// * {byte_length_} is always lower or equal than the amount of memory with
// permissions set to kReadWrite;
// * This is guaranteed by incrementing {byte_length_} with a
// compare_exchange after changing the permissions.
// * This invariant is the reason why we cannot use a fetch_add.
DCHECK(is_wasm_memory());
max_pages = std::min(max_pages, byte_capacity_ / wasm::kWasmPageSize);
// Do a compare-exchange loop, because we also need to adjust page
// permissions. Note that multiple racing grows both try to set page
// permissions for the entire range (to be RW), so the operating system
// should deal with that raciness. We know we succeeded when we can
// compare/swap the old length with the new length.
size_t old_length = byte_length_.load(std::memory_order_relaxed);
if (delta_pages == 0) {
return {old_length / wasm::kWasmPageSize}; // degenerate grow.
}
if (delta_pages > max_pages) return {}; // would never work.
size_t new_length = 0;
while (true) {
size_t current_pages = old_length / wasm::kWasmPageSize;
// Check if we have exceed the supplied maximum.
if (current_pages > (max_pages - delta_pages)) return {};
new_length = (current_pages + delta_pages) * wasm::kWasmPageSize;
// Try to adjust the permissions on the memory.
if (!i::SetPermissions(GetPlatformPageAllocator(), buffer_start_,
new_length, PageAllocator::kReadWrite)) {
// This is a nondeterministic failure; mark as such in the WasmEngine (for
// differential fuzzing).
wasm::WasmEngine::set_had_nondeterminism();
return {};
}
if (byte_length_.compare_exchange_weak(old_length, new_length,
std::memory_order_acq_rel)) {
// Successfully updated both the length and permissions.
break;
}
}
return {old_length / wasm::kWasmPageSize};
}
void BackingStore::BroadcastSharedWasmMemoryGrow(Isolate* isolate) const {
GlobalBackingStoreRegistry::BroadcastSharedWasmMemoryGrow(isolate, this);
}
void BackingStore::RemoveSharedWasmMemoryObjects(Isolate* isolate) {
GlobalBackingStoreRegistry::Purge(isolate);
}
void BackingStore::UpdateSharedWasmMemoryObjects(Isolate* isolate) {
GlobalBackingStoreRegistry::UpdateSharedWasmMemoryObjects(isolate);
}
void BackingStore::MakeWasmMemoryResizableByJS(ResizableFlag resizable) {
DCHECK(is_wasm_memory());
// Shared memory does not update this flag, because different ABs may share
// the backing store but with different resizability.
DCHECK(!is_shared());
if (resizable) {
set_flag(kIsResizableByJs);
} else {
clear_flag(kIsResizableByJs);
}
}
#endif // V8_ENABLE_WEBASSEMBLY
// Commit already reserved memory (for RAB backing stores (not shared)).
BackingStore::ResizeOrGrowResult BackingStore::ResizeInPlace(
Isolate* isolate, size_t new_byte_length) {
size_t page_size = AllocatePageSize();
size_t new_committed_pages;
bool round_return_value =
RoundUpToPageSize(new_byte_length, page_size,
JSArrayBuffer::kMaxByteLength, &new_committed_pages);
CHECK(round_return_value);
size_t new_committed_length = new_committed_pages * page_size;
DCHECK_LE(new_byte_length, new_committed_length);
DCHECK(!is_shared());
// Validate the new length against the out-of-sandbox capacity to prevent
// an in-sandbox corruption from bypassing bounds checks.
SBXCHECK_LE(new_byte_length, max_byte_length_);
if (new_byte_length < byte_length_) {
// Zero the memory so that in case the buffer is grown later, we have
// zeroed the contents already. This is especially needed for the portion of
// the memory we're not going to decommit below (since it belongs to a
// committed page). In addition, we don't rely on all platforms always
// zeroing decommitted-then-recommitted memory, but zero the memory
// explicitly here.
memset(reinterpret_cast<uint8_t*>(buffer_start_) + new_byte_length, 0,
byte_length_ - new_byte_length);
// Check if we can un-commit some pages.
size_t old_committed_pages;
round_return_value =
RoundUpToPageSize(byte_length_, page_size,
JSArrayBuffer::kMaxByteLength, &old_committed_pages);
CHECK(round_return_value);
DCHECK_LE(new_committed_pages, old_committed_pages);
if (new_committed_pages < old_committed_pages) {
size_t old_committed_length = old_committed_pages * page_size;
if (!i::SetPermissions(
GetPlatformPageAllocator(),
reinterpret_cast<uint8_t*>(buffer_start_) + new_committed_length,
old_committed_length - new_committed_length,
PageAllocator::kNoAccess)) {
return kFailure;
}
}
// Changing the byte length wouldn't strictly speaking be needed, since
// the JSArrayBuffer already stores the updated length. This is to keep
// the BackingStore and JSArrayBuffer in sync.
byte_length_ = new_byte_length;
return kSuccess;
}
if (new_byte_length == byte_length_) {
// i::SetPermissions with size 0 fails on some platforms, so special
// handling for the case byte_length_ == new_byte_length == 0 is required.
return kSuccess;
}
// Try to adjust the permissions on the memory.
if (!i::SetPermissions(GetPlatformPageAllocator(), buffer_start_,
new_committed_length, PageAllocator::kReadWrite)) {
return kFailure;
}
byte_length_ = new_byte_length;
return kSuccess;
}
// Commit already reserved memory (for GSAB backing stores (shared)).
BackingStore::ResizeOrGrowResult BackingStore::GrowInPlace(
Isolate* isolate, size_t new_byte_length) {
size_t page_size = AllocatePageSize();
size_t new_committed_pages;
bool round_return_value =
RoundUpToPageSize(new_byte_length, page_size,
JSArrayBuffer::kMaxByteLength, &new_committed_pages);
CHECK(round_return_value);
size_t new_committed_length = new_committed_pages * page_size;
DCHECK_LE(new_byte_length, new_committed_length);
DCHECK(is_shared());
// Validate the new length against the out-of-sandbox capacity to prevent
// an in-sandbox corruption from bypassing bounds checks.
SBXCHECK_LE(new_byte_length, max_byte_length_);
// See comment in GrowWasmMemoryInPlace.
// GrowableSharedArrayBuffer.prototype.grow can be called from several
// threads. If two threads try to grow() in a racy way, the spec allows the
// larger grow to throw also if the smaller grow succeeds first. The
// implementation below doesn't throw in that case - instead, it retries and
// succeeds. If the larger grow finishes first though, the smaller grow must
// throw.
size_t old_byte_length = byte_length_.load(std::memory_order_seq_cst);
while (true) {
if (new_byte_length < old_byte_length) {
// The caller checks for the new_byte_length < old_byte_length_ case. This
// can only happen if another thread grew the memory after that.
return kRace;
}
if (new_byte_length == old_byte_length) {
// i::SetPermissions with size 0 fails on some platforms, so special
// handling for the case old_byte_length == new_byte_length == 0 is
// required.
return kSuccess;
}
// Try to adjust the permissions on the memory.
if (!i::SetPermissions(GetPlatformPageAllocator(), buffer_start_,
new_committed_length, PageAllocator::kReadWrite)) {
return kFailure;
}
// compare_exchange_weak updates old_byte_length.
if (byte_length_.compare_exchange_weak(old_byte_length, new_byte_length,
std::memory_order_seq_cst)) {
// Successfully updated both the length and permissions.
break;
}
}
return kSuccess;
}
std::unique_ptr<BackingStore> BackingStore::WrapAllocation(
void* allocation_base, size_t allocation_length,
v8::BackingStore::DeleterCallback deleter, void* deleter_data,
SharedFlag shared) {
bool is_empty_deleter = (deleter == v8::BackingStore::EmptyDeleter);
auto result =
new BackingStore(allocation_base, // start
allocation_length, // length
allocation_length, // max length
allocation_length, // capacity
shared, ResizableFlag{false}, ImmutableFlag{false},
WasmMemoryFlag::kNotWasm, HasGuardRegions{false},
CustomDeleter{true}, EmptyDeleter{is_empty_deleter});
result->type_specific_data_.deleter = {deleter, deleter_data};
TRACE_BS("BS:wrap bs=%p mem=%p (length=%zu)\n", result,
result->buffer_start(), result->byte_length());
return std::unique_ptr<BackingStore>(result);
}
std::unique_ptr<BackingStore> BackingStore::EmptyBackingStore(
SharedFlag shared) {
auto result =
new BackingStore(nullptr, // start
0, // length
0, // max length
0, // capacity
shared, ResizableFlag{false}, ImmutableFlag{false},
WasmMemoryFlag::kNotWasm, HasGuardRegions{false},
CustomDeleter{false}, EmptyDeleter{false});
return std::unique_ptr<BackingStore>(result);
}
v8::ArrayBuffer::Allocator* BackingStore::get_v8_api_array_buffer_allocator() {
CHECK(!is_wasm_memory());
auto array_buffer_allocator =
holds_shared_ptr_to_allocator()
? type_specific_data_.v8_api_array_buffer_allocator_shared.get()
: type_specific_data_.v8_api_array_buffer_allocator;
CHECK_NOT_NULL(array_buffer_allocator);
return array_buffer_allocator;
}
SharedWasmMemoryData* BackingStore::get_shared_wasm_memory_data() const {
CHECK(is_wasm_memory() && is_shared());
auto shared_wasm_memory_data = type_specific_data_.shared_wasm_memory_data;
CHECK(shared_wasm_memory_data);
return shared_wasm_memory_data;
}
namespace {
// Implementation details of GlobalBackingStoreRegistry.
struct GlobalBackingStoreRegistryImpl {
GlobalBackingStoreRegistryImpl() = default;
base::Mutex mutex_;
std::unordered_map<const void*, std::weak_ptr<BackingStore>> map_;
};
DEFINE_LAZY_LEAKY_OBJECT_GETTER(GlobalBackingStoreRegistryImpl,
GetGlobalBackingStoreRegistryImpl)
} // namespace
void GlobalBackingStoreRegistry::Unregister(BackingStore* backing_store) {
if (!backing_store->globally_registered()) return;
CHECK(backing_store->is_wasm_memory());
DCHECK_NOT_NULL(backing_store->buffer_start());
GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();
base::MutexGuard scope_lock(&impl->mutex_);
const auto& result = impl->map_.find(backing_store->buffer_start());
if (result != impl->map_.end()) {
DCHECK(!result->second.lock());
impl->map_.erase(result);
}
backing_store->clear_flag(BackingStore::kGloballyRegistered);
}
void GlobalBackingStoreRegistry::Purge(Isolate* isolate) {
// We need to keep a reference to all backing stores that are inspected
// in the purging loop below. Otherwise, we might get a deadlock
// if the temporary backing store reference created in the loop is
// the last reference. In that case the destructor of the backing store
// may try to take the &impl->mutex_ in order to unregister itself.
std::vector<std::shared_ptr<BackingStore>> prevent_destruction_under_lock;
GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();
base::MutexGuard scope_lock(&impl->mutex_);
// Purge all entries in the map that refer to the given isolate.
for (auto& entry : impl->map_) {
auto backing_store = entry.second.lock();
prevent_destruction_under_lock.emplace_back(backing_store);
if (!backing_store) continue; // skip entries where weak ptr is null
CHECK(backing_store->is_wasm_memory());
if (!backing_store->is_shared()) continue; // skip non-shared memory
SharedWasmMemoryData* shared_data =
backing_store->get_shared_wasm_memory_data();
// Remove this isolate from the isolates list.
std::vector<Isolate*>& isolates = shared_data->isolates_;
auto isolates_it = std::find(isolates.begin(), isolates.end(), isolate);
if (isolates_it != isolates.end()) {
*isolates_it = isolates.back();
isolates.pop_back();
}
DCHECK_EQ(isolates.end(),
std::find(isolates.begin(), isolates.end(), isolate));
}
}
#if V8_ENABLE_WEBASSEMBLY
void GlobalBackingStoreRegistry::AddSharedWasmMemoryObject(
Isolate* isolate, std::shared_ptr<BackingStore> backing_store,
DirectHandle<WasmMemoryObject> memory_object) {
DCHECK(backing_store->is_wasm_memory());
DCHECK(backing_store->is_shared());
// Add to the weak array list of shared memory objects in the isolate.
isolate->AddSharedWasmMemory(memory_object);
GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();
base::MutexGuard scope_lock(&impl->mutex_);
// Register this backing store globally.
if (!backing_store->globally_registered() && backing_store->buffer_start()) {
TRACE_BS("BS:reg bs=%p mem=%p (length=%zu, capacity=%zu)\n",
backing_store.get(), backing_store->buffer_start(),
backing_store->byte_length(), backing_store->byte_capacity());
std::weak_ptr<BackingStore> weak = backing_store;
auto result = impl->map_.insert({backing_store->buffer_start(), weak});
CHECK(result.second);
backing_store->set_flag(BackingStore::kGloballyRegistered);
}
// Add the isolate to the list of isolates sharing this backing store.
SharedWasmMemoryData* shared_data =
backing_store->get_shared_wasm_memory_data();
auto& isolates = shared_data->isolates_;
int free_entry = -1;
for (size_t i = 0; i < isolates.size(); i++) {
if (isolates[i] == isolate) return;
if (isolates[i] == nullptr) free_entry = static_cast<int>(i);
}
if (free_entry >= 0) {
isolates[free_entry] = isolate;
} else {
isolates.push_back(isolate);
}
}
void GlobalBackingStoreRegistry::BroadcastSharedWasmMemoryGrow(
Isolate* isolate, const BackingStore* backing_store) {
{
GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();
// The global lock protects the list of isolates per backing store.
base::MutexGuard scope_lock(&impl->mutex_);
SharedWasmMemoryData* shared_data =
backing_store->get_shared_wasm_memory_data();
for (Isolate* other : shared_data->isolates_) {
if (other == isolate) continue;
other->stack_guard()->RequestGrowSharedMemory();
}
}
// Update memory objects in this isolate.
UpdateSharedWasmMemoryObjects(isolate);
}
void GlobalBackingStoreRegistry::UpdateSharedWasmMemoryObjects(
Isolate* isolate) {
// We call here from the stack guard at loop back edges, where we don't want
// GC to get in the way of loop-related compiler optimizations.
// TODO(clemensb): This prevents us from updating the external memory via
// memory_object->managed_backing_store()->UpdateEstimatedSize(...). Hence we
// report too little memory for shared memory in other isolates. We might need
// a second interrupt type just for updating the external memory.
DisallowHeapAllocation no_gc;
SealHandleScope seal_handle_scope{isolate};
Tagged<WeakArrayList> shared_wasm_memories =
Cast<WeakArrayList>(isolate->root(RootIndex::kSharedWasmMemories));
const uint32_t shared_wasm_memories_len =
shared_wasm_memories->length().value();
for (uint32_t i = 0; i < shared_wasm_memories_len; ++i) {
Tagged<HeapObject> obj;
if (!shared_wasm_memories->Get(i).GetHeapObject(&obj)) continue;
Tagged<WasmMemoryObject> memory_object = Cast<WasmMemoryObject>(obj);
memory_object->UpdateInstances(isolate);
if (Tagged<JSArrayBuffer> shared_ab;
TryCast<JSArrayBuffer>(memory_object->array_buffer(), &shared_ab) &&
!shared_ab->is_resizable_by_js()) {
// Clear the JSArrayBuffer such that we allocate a fresh one on the next
// access. Check that the stored backing store is correct, because that
// will be used to create the new JSArrayBuffer.
// Only refresh array buffers whose associated Wasm memory actually grew.
DCHECK_EQ(shared_ab->GetBackingStore(), memory_object->backing_store());
if (shared_ab->byte_length() <
memory_object->backing_store()->byte_length()) {
memory_object->set_array_buffer(
ReadOnlyRoots{isolate}.undefined_value());
}
}
}
}
#endif // V8_ENABLE_WEBASSEMBLY
} // namespace v8::internal
#undef TRACE_BS