// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <optional>
#include "src/builtins/builtins-inl.h"
#include "src/builtins/data-view-ops.h"
#include "src/common/assert-scope.h"
#include "src/common/message-template.h"
#include "src/compiler/wasm-compiler.h"
#include "src/debug/debug.h"
#include "src/deoptimizer/deoptimizer.h"
#include "src/execution/arguments-inl.h"
#include "src/execution/frames-inl.h"
#include "src/execution/frames.h"
#include "src/handles/handles.h"
#include "src/heap/factory.h"
#include "src/heap/read-only-heap.h"
#include "src/numbers/conversions.h"
#include "src/objects/dictionary-inl.h"
#include "src/objects/js-promise-inl.h"
#include "src/objects/lookup-inl.h"
#include "src/objects/managed-inl.h"
#include "src/objects/object-conversions-inl.h"
#include "src/objects/object-list-macros.h"
#include "src/objects/objects-inl.h"
#include "src/objects/property-descriptor.h"
#include "src/runtime/runtime-utils.h"
#include "src/strings/unicode-inl.h"
#include "src/trap-handler/trap-handler.h"
#include "src/wasm/module-compiler.h"
#include "src/wasm/signature-hashing.h"
#include "src/wasm/value-type.h"
#include "src/wasm/wasm-code-manager.h"
#include "src/wasm/wasm-constants.h"
#include "src/wasm/wasm-debug.h"
#include "src/wasm/wasm-engine-globals.h"
#include "src/wasm/wasm-export-wrapper-cache.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-opcodes-inl.h"
#include "src/wasm/wasm-stack-wrapper-cache.h"
#include "src/wasm/wasm-subtyping.h"
#include "src/wasm/wasm-value.h"
#if V8_ENABLE_WEBASSEMBLY && V8_ENABLE_DRUMBRAKE
#include "src/wasm/interpreter/wasm-interpreter.h"
#endif // V8_ENABLE_WEBASSEMBLY && V8_ENABLE_DRUMBRAKE
namespace v8::internal {
// TODO(13036): See if we can find a way to have the stack walker visit
// tagged values being passed from Wasm to runtime functions. In the meantime,
// disallow access to safe-looking-but-actually-unsafe stack-backed handles
// and thereby force manual creation of safe handles (backed by HandleScope).
class RuntimeArgumentsWithoutHandles : public RuntimeArguments {
public:
RuntimeArgumentsWithoutHandles(int length, Address* arguments)
: RuntimeArguments(length, arguments) {}
private:
// Disallowing the superclass method.
template <class S = Object>
V8_INLINE DirectHandle<S> at(int index) const;
};
#define RuntimeArguments RuntimeArgumentsWithoutHandles
// (End of TODO(13036)-related hackery.)
namespace {
template <typename FrameType>
class FrameFinder {
public:
explicit FrameFinder(Isolate* isolate,
std::initializer_list<StackFrame::Type>
skipped_frame_types = {StackFrame::EXIT})
: frame_iterator_(isolate, isolate->thread_local_top(),
StackFrameIterator::FirstStackOnly{}) {
// We skip at least one frame.
DCHECK_LT(0, skipped_frame_types.size());
for (auto type : skipped_frame_types) {
DCHECK_EQ(type, frame_iterator_.frame()->type());
USE(type);
frame_iterator_.Advance();
}
// Type check the frame where the iterator stopped now.
DCHECK_NOT_NULL(frame());
}
FrameType* frame() { return FrameType::cast(frame_iterator_.frame()); }
private:
StackFrameIterator frame_iterator_;
};
Tagged<Object> ThrowWasmError(
Isolate* isolate, MessageTemplate message,
std::initializer_list<DirectHandle<Object>> args = {}) {
#if V8_ENABLE_DRUMBRAKE
if (v8_flags.wasm_jitless) {
// Store the trap reason to be retrieved later when the interpreter will
// trap while detecting the thrown exception.
wasm::WasmInterpreterThread::SetRuntimeLastWasmError(isolate, message);
}
#endif // V8_ENABLE_DRUMBRAKE
DirectHandle<JSObject> error_obj =
isolate->factory()->NewWasmRuntimeError(message, base::VectorOf(args));
JSObject::AddProperty(isolate, error_obj,
isolate->factory()->wasm_uncatchable_symbol(),
isolate->factory()->true_value(), NONE);
return isolate->Throw(*error_obj);
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmWasmToJSObject) {
HandleScope handle_scope(isolate);
DCHECK_EQ(1, args.length());
DirectHandle<Object> value(args[0], isolate);
return *wasm::WasmToJSObject(isolate, value);
}
// Parameters:
// args[0]: the object, any JS value.
// args[1]: the expected canonicalized ValueType, Smi-tagged.
// Type checks the object against the type; if the check succeeds, returns the
// object in its wasm representation; otherwise throws a type error.
RUNTIME_FUNCTION(Runtime_WasmJSToWasmObject) {
HandleScope scope(isolate);
DCHECK_EQ(2, args.length());
DirectHandle<Object> value(args[0], isolate);
// Make sure ValueType fits properly in a Smi.
static_assert(wasm::CanonicalValueType::kLastUsedBit + 1 <= kSmiValueSize);
int raw_type = args.smi_value_at(1);
wasm::CanonicalValueType expected =
wasm::CanonicalValueType::FromRawBitField(raw_type);
const char* error_message;
DirectHandle<Object> result;
bool success = JSToWasmObject(isolate, value, expected, &error_message)
.ToHandle(&result);
Tagged<Object> ret = success
? *result
: isolate->Throw(*isolate->factory()->NewTypeError(
MessageTemplate::kWasmTrapJSTypeError));
return ret;
}
RUNTIME_FUNCTION(Runtime_WasmMemoryGrow) {
HandleScope scope(isolate);
DCHECK_EQ(3, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
// {memory_index} and {delta_pages} are checked to be positive Smis in the
// WasmMemoryGrow builtin which calls this runtime function.
uint32_t memory_index = args.positive_smi_value_at(1);
uint32_t delta_pages = args.positive_smi_value_at(2);
DirectHandle<WasmMemoryObject> memory_object{
trusted_instance_data->memory_object(memory_index), isolate};
int ret = WasmMemoryObject::Grow(isolate, memory_object, delta_pages);
// The WasmMemoryGrow builtin which calls this runtime function expects us to
// always return a Smi.
DCHECK(!isolate->has_exception() ||
IsTerminationException(isolate->exception()));
return Smi::FromInt(ret);
}
RUNTIME_FUNCTION(Runtime_TrapHandlerThrowWasmError) {
CHECK(isolate->IsOnCentralStack());
HandleScope scope(isolate);
FrameFinder<WasmFrame> frame_finder(isolate, {StackFrame::EXIT});
WasmFrame* frame = frame_finder.frame();
int pos = frame->position();
wasm::WasmCodeRefScope code_ref_scope;
auto wire_bytes = frame->wasm_code()->native_module()->wire_bytes();
wasm::WasmOpcode op = static_cast<wasm::WasmOpcode>(wire_bytes.at(pos));
MessageTemplate message = MessageTemplate::kWasmTrapMemOutOfBounds;
if (op == wasm::kGCPrefix || op == wasm::kExprRefAsNonNull ||
op == wasm::kExprCallRef || op == wasm::kExprReturnCallRef ||
// Calling imported string function with null can trigger a signal.
op == wasm::kExprCallFunction || op == wasm::kExprReturnCall) {
message = MessageTemplate::kWasmTrapNullDereference;
} else if (op == wasm::kAtomicPrefix) {
op = wasm::Decoder{wire_bytes}
.read_prefixed_opcode<wasm::Decoder::NoValidationTag>(
&wire_bytes.begin()[pos])
.first;
// shared-everything atomic instructions.
if (op >= 0xFE4F || op == wasm::kExprArrayWait) {
message = MessageTemplate::kWasmTrapNullDereference;
}
#define CASE(name, ...) || op == wasm::kExpr##name
DCHECK_EQ(op >= 0xFE4F || op == wasm::kExprArrayWait,
false FOREACH_ATOMIC_GC_OPCODE(CASE));
#undef CASE
}
return ThrowWasmError(isolate, message);
}
RUNTIME_FUNCTION(Runtime_ThrowWasmError) {
DCHECK(isolate->IsOnCentralStack());
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
int message_id = args.smi_value_at(0);
return ThrowWasmError(isolate, MessageTemplateFromInt(message_id));
}
RUNTIME_FUNCTION(Runtime_ThrowWasmStackOverflow) {
SealHandleScope shs(isolate);
DCHECK_LE(0, args.length());
return isolate->StackOverflow();
}
RUNTIME_FUNCTION(Runtime_WasmThrowJSTypeError) {
HandleScope scope(isolate);
DCHECK_EQ(0, args.length());
THROW_NEW_ERROR_RETURN_FAILURE(
isolate, NewTypeError(MessageTemplate::kWasmTrapJSTypeError));
}
RUNTIME_FUNCTION(Runtime_ThrowWasmJSPISuspendError) {
HandleScope scope(isolate);
DCHECK_EQ(0, args.length());
MessageTemplate message = MessageTemplate::kWasmSuspendJSFrames;
Tagged<WasmSuspenderObject> suspender =
isolate->isolate_data()->active_suspender();
if (IsUndefined(suspender->resume())) {
// We are not within a valid "WebAssembly.promising" scope. This is either a
// suspender created for a non-promising export call under the stress mode,
// or the sentinel suspender representing the initial stack.
// Throw with a more precise error message.
message = MessageTemplate::kWasmSuspendError;
}
DirectHandle<JSObject> error_obj =
isolate->factory()->NewWasmSuspendError(message);
return isolate->Throw(*error_obj);
}
RUNTIME_FUNCTION(Runtime_ThrowWasmFXSuspendError) {
HandleScope scope(isolate);
DCHECK_EQ(0, args.length());
MessageTemplate message = MessageTemplate::kWasmFXSuspendError;
DirectHandle<JSObject> error_obj =
isolate->factory()->NewWasmSuspendError(message);
return isolate->Throw(*error_obj);
}
RUNTIME_FUNCTION(Runtime_WasmThrowRangeError) {
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
MessageTemplate message_id = MessageTemplateFromInt(args.smi_value_at(0));
THROW_NEW_ERROR_RETURN_FAILURE(isolate, NewRangeError(message_id));
}
RUNTIME_FUNCTION(Runtime_WasmThrowDataViewTypeError) {
HandleScope scope(isolate);
DCHECK_EQ(2, args.length());
MessageTemplate message_id = MessageTemplateFromInt(args.smi_value_at(0));
DataViewOp op = static_cast<DataViewOp>(isolate->error_message_param());
DirectHandle<String> op_name =
isolate->factory()->NewStringFromAsciiChecked(ToString(op));
DirectHandle<Object> value(args[1], isolate);
THROW_NEW_ERROR_RETURN_FAILURE(isolate,
NewTypeError(message_id, op_name, value));
}
RUNTIME_FUNCTION(Runtime_WasmThrowDataViewDetachedError) {
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
MessageTemplate message_id = MessageTemplateFromInt(args.smi_value_at(0));
DataViewOp op = static_cast<DataViewOp>(isolate->error_message_param());
DirectHandle<String> op_name =
isolate->factory()->NewStringFromAsciiChecked(ToString(op));
THROW_NEW_ERROR_RETURN_FAILURE(isolate, NewTypeError(message_id, op_name));
}
RUNTIME_FUNCTION(Runtime_WasmThrowTypeError) {
HandleScope scope(isolate);
DCHECK_EQ(2, args.length());
MessageTemplate message_id = MessageTemplateFromInt(args.smi_value_at(0));
DirectHandle<Object> arg(args[1], isolate);
if (IsSmi(*arg)) {
THROW_NEW_ERROR_RETURN_FAILURE(isolate, NewTypeError(message_id));
} else {
THROW_NEW_ERROR_RETURN_FAILURE(isolate, NewTypeError(message_id, arg));
}
}
RUNTIME_FUNCTION(Runtime_WasmThrow) {
HandleScope scope(isolate);
DCHECK_EQ(3, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[2]);
Tagged<Context> context = trusted_instance_data->native_context();
isolate->set_context(context);
DirectHandle<WasmExceptionTag> tag(Cast<WasmExceptionTag>(args[0]), isolate);
DirectHandle<FixedArray> values(Cast<FixedArray>(args[1]), isolate);
auto js_tag = Cast<WasmTagObject>(context->wasm_js_tag());
if (*tag == js_tag->tag()) {
return isolate->Throw(values->get(0));
} else {
DirectHandle<WasmExceptionPackage> exception =
WasmExceptionPackage::New(isolate, tag, values);
return isolate->Throw(*exception);
}
}
RUNTIME_FUNCTION(Runtime_WasmReThrow) {
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
return isolate->ReThrow(args[0]);
}
RUNTIME_FUNCTION(Runtime_WasmStackGuard) {
SealHandleScope shs(isolate);
DCHECK_EQ(1, args.length());
TRACE_EVENT("v8.execute", "V8.StackGuard");
uint32_t gap = args.positive_smi_value_at(0);
// Check if this is a real stack overflow.
StackLimitCheck check(isolate);
if (check.WasmHasOverflowed(gap)) return isolate->StackOverflow();
return isolate->stack_guard()->HandleInterrupts(
StackGuard::InterruptLevel::kAnyEffect);
}
// For loop back edges in optimized code. Avoids triggering side effects that
// could get in the way of optimizations, and doesn't need to check for real
// stack overflows because loops don't change the stack height.
// Note: API interrupts for debugging purposes can execute arbitrary JS,
// and we don't guard against that here. So for very particular (and presumably
// very unlikely) circumstances, debugging sessions can cause crashes.
// To properly fix that, we should implement lazy-deopt support for Wasm.
RUNTIME_FUNCTION(Runtime_WasmStackGuardLoop) {
DCHECK_EQ(0, args.length());
SealHandleScope shs(isolate);
TRACE_EVENT("v8.execute", "V8.StackGuard");
return isolate->stack_guard()->HandleInterrupts(
StackGuard::InterruptLevel::kNoHeapWrites);
}
RUNTIME_FUNCTION(Runtime_WasmCompileLazy) {
SealHandleScope shs(isolate);
DCHECK_EQ(2, args.length());
int func_index = args.smi_value_at(1);
// Note: This runtime function *must not* cause a GC, because the calling
// builtin (also called "WasmCompileLazy") spilled all arguments to the call
// but those are never visited by GC.
DisallowGarbageCollection no_gc;
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
TRACE_EVENT("v8.wasm", "wasm.CompileLazy", "func_index", func_index);
// A raw pointer is fine here, as the native module is kept alive by the
// caller implicitly (via the `WasmTrustedInstanceData`).
wasm::NativeModule* native_module = trusted_instance_data->native_module();
DCHECK(isolate->context().is_null());
DCHECK(trusted_instance_data->has_native_context());
isolate->set_context(trusted_instance_data->native_context());
wasm::CompileLazy(isolate, native_module, func_index);
native_module->counter_updates()->Publish(isolate);
return Smi::FromInt(
wasm::JumpTableOffset(native_module->module(), func_index));
}
namespace {
Tagged<FixedArray> AllocateFeedbackVector(
Isolate* isolate,
DirectHandle<WasmTrustedInstanceData> trusted_instance_data,
int declared_func_index) {
DCHECK(isolate->context().is_null());
isolate->set_context(trusted_instance_data->native_context());
const wasm::WasmModule* module =
trusted_instance_data->native_module()->module();
int func_index = declared_func_index + module->num_imported_functions;
int num_slots = NumFeedbackSlots(module, func_index);
DirectHandle<FixedArray> vector =
isolate->factory()->NewFixedArrayWithZeroes(num_slots);
DCHECK_EQ(trusted_instance_data->feedback_vectors()->get(declared_func_index),
Smi::zero());
trusted_instance_data->feedback_vectors()->set(declared_func_index, *vector);
isolate->set_context({});
return *vector;
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmAllocateFeedbackVector) {
DCHECK(isolate->IsOnCentralStack());
HandleScope scope(isolate);
DCHECK_EQ(3, args.length());
DCHECK(v8_flags.wasm_inlining);
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
int declared_func_index = args.smi_value_at(1);
wasm::NativeModule** native_module_stack_slot =
reinterpret_cast<wasm::NativeModule**>(args.address_of_arg_at(2));
wasm::NativeModule* native_module = trusted_instance_data->native_module();
// We have to save the native_module on the stack, in case the allocation
// triggers a GC and we need the module to scan LiftoffSetupFrame stack frame.
*native_module_stack_slot = native_module;
return AllocateFeedbackVector(isolate, trusted_instance_data,
declared_func_index);
}
RUNTIME_FUNCTION(Runtime_WasmLiftoffDeoptFinish) {
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
// Destroy the Deoptimizer object stored on the isolate.
size_t deopt_frame_count = Deoptimizer::DeleteForWasm(isolate);
size_t i = 0;
// For each liftoff frame, replace the smi in the feedback vector slot holding
// the declared function index with the actual feedback vector from the
// WasmTrustedInstanceData. If it doesn't exist yet, it needs to be allocated
// (which is also the reason why this wasn't done by the deoptimizer already.)
for (StackFrameIterator it(isolate); !it.done(); it.Advance()) {
StackFrame* frame = it.frame();
if (frame->is_wasm() && WasmFrame::cast(frame)->wasm_code()->is_liftoff()) {
Address vector_address =
frame->fp() - WasmLiftoffFrameConstants::kFeedbackVectorOffset;
Tagged<Object> feedback_slot_value(Memory<intptr_t>(vector_address));
CHECK(feedback_slot_value.IsSmi());
int declared_func_index = Cast<Smi>(feedback_slot_value).value();
Tagged<Object> vector =
trusted_instance_data->feedback_vectors()->get(declared_func_index);
// If the vector doesn't exist, allocate a new feedback vector. This
// happens if the corresponding function wasn't executed in Liftoff yet
// for this particular module instantiation.
if (vector == Smi::zero()) {
if (v8_flags.trace_deopt_verbose) {
wasm::WasmCodeRefScope code_ref_scope;
const int module_func_index =
trusted_instance_data->native_module()->num_imported_functions() +
declared_func_index;
const wasm::WasmCode* code =
trusted_instance_data->native_module()->GetCode(
module_func_index);
PrintF(
"Wasm deoptimization: allocating feedback vector for function %s "
"[%d]\n",
code ? code->DebugName().c_str() : "<no code object>",
module_func_index);
}
vector = AllocateFeedbackVector(isolate, trusted_instance_data,
declared_func_index);
}
memcpy(reinterpret_cast<void*>(vector_address), &vector,
sizeof(intptr_t));
if (++i == deopt_frame_count) {
break; // All deopt frames have been visited.
}
}
}
return ReadOnlyRoots(isolate).undefined_value();
}
namespace {
void ReplaceJSToWasmWrapper(
Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data,
int function_index, Tagged<Code> wrapper_code) {
Tagged<WasmFuncRef> func_ref;
// Always expect a func_ref. If this fails, we are maybe compiling a wrapper
// for the start function. This function is only called once, so this should
// not happen.
CHECK(trusted_instance_data->try_get_func_ref(function_index, &func_ref));
Tagged<JSFunction> external_function;
CHECK(func_ref->internal(isolate)->try_get_external(&external_function));
CHECK(external_function->shared()->HasWasmExportedFunctionData(isolate));
external_function->UpdateCode(isolate, wrapper_code);
}
} // namespace
RUNTIME_FUNCTION(Runtime_TierUpJSToWasmWrapper) {
DCHECK_EQ(1, args.length());
// Avoid allocating a HandleScope and handles on the fast path.
Tagged<WasmExportedFunctionData> function_data =
SbxCast<WasmExportedFunctionData>(TrustedCast<TrustedObject>(args[0]));
Tagged<WasmTrustedInstanceData> trusted_data = function_data->instance_data();
const wasm::WasmModule* module = trusted_data->module();
const int function_index = function_data->function_index();
const wasm::WasmFunction& function = module->functions[function_index];
const wasm::CanonicalTypeIndex sig_id =
module->canonical_sig_id(function.sig_index);
const wasm::CanonicalSig* sig =
wasm::GetTypeCanonicalizer()->LookupFunctionSignature(sig_id);
Tagged<CodeWrapper> maybe_cached_wrapper =
wasm::WasmExportWrapperCache::Get(isolate, sig_id);
Tagged<Code> wrapper_code;
if (!maybe_cached_wrapper.is_null()) {
wrapper_code = maybe_cached_wrapper->code(isolate);
} else {
// Set the context on the isolate and open a handle scope for allocation of
// new objects. Wrap {trusted_data} in a handle so it survives GCs.
DCHECK(isolate->context().is_null());
isolate->set_context(trusted_data->native_context());
HandleScope scope(isolate);
DirectHandle<WasmTrustedInstanceData> trusted_data_handle{trusted_data,
isolate};
DirectHandle<Code> new_wrapper_code =
wasm::JSToWasmWrapperCompilationUnit::CompileJSToWasmWrapper(isolate,
sig);
// Compilation must have installed the wrapper into the cache.
DCHECK_EQ(new_wrapper_code->wrapper(),
wasm::WasmExportWrapperCache::Get(isolate, sig_id));
// Reset raw pointers still needed outside the slow path.
wrapper_code = *new_wrapper_code;
trusted_data = *trusted_data_handle;
function_data = {};
}
// Replace the wrapper for the function that triggered the tier-up.
// This is to ensure that the wrapper is replaced, even if the function
// is implicitly exported and is not part of the export_table.
ReplaceJSToWasmWrapper(isolate, trusted_data, function_index, wrapper_code);
// Iterate over all exports to replace eagerly the wrapper for all functions
// that share the signature of the function that tiered up.
for (wasm::WasmExport exp : module->export_table) {
if (exp.kind != wasm::kExternalFunction) continue;
int index = static_cast<int>(exp.index);
if (index == function_index) continue; // Already replaced.
const wasm::WasmFunction& exp_function = module->functions[index];
if (module->canonical_sig_id(exp_function.sig_index) != sig_id) {
continue; // Different signature.
}
ReplaceJSToWasmWrapper(isolate, trusted_data, index, wrapper_code);
}
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_IsWasmExternalFunction) {
SealHandleScope shs(isolate);
DisallowGarbageCollection no_gc;
DCHECK_EQ(1, args.length());
return isolate->heap()->ToBoolean(
WasmExternalFunction::IsWasmExternalFunction(args[0]));
}
RUNTIME_FUNCTION(Runtime_TierUpWasmToJSWrapper) {
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
DirectHandle<WasmImportData> import_data(
SbxCast<WasmImportData>(TrustedCast<TrustedObject>(args[0])), isolate);
DCHECK(isolate->context().is_null());
isolate->set_context(import_data->native_context());
const wasm::CanonicalSig* sig = import_data->sig();
wasm::WasmCodeRefScope code_ref_scope;
// Compile a wrapper for the target callable.
DirectHandle<JSReceiver> callable(Cast<JSReceiver>(import_data->callable()),
isolate);
wasm::Suspend suspend = import_data->suspend();
// We don't need to care about exactness of the import here, because that
// has already been validated (hence no kLinkError can happen here).
wasm::CanonicalValueType expected_type = wasm::CanonicalValueType::Ref(
sig->index(), SharedFlag{false}, wasm::RefTypeKind::kFunction);
wasm::ResolvedWasmImport resolved({}, -1, callable, expected_type, sig,
wasm::WellKnownImport::kUninstantiated);
wasm::ImportCallKind kind = resolved.kind();
callable = resolved.callable(); // Update to ultimate target.
DCHECK_NE(wasm::ImportCallKind::kLinkError, kind);
int expected_arity = static_cast<int>(sig->parameter_count());
if (kind == wasm::ImportCallKind::kJSFunction) {
expected_arity = Cast<JSFunction>(callable)
->shared()
->internal_formal_parameter_count_without_receiver();
}
// Lookup or compile a wrapper. The returned wrapper handle is not needed
// because GetCompiled updates the code pointer table entry in-place.
wasm::WasmImportWrapperCache* cache = wasm::GetWasmImportWrapperCache();
cache->GetCompiled(isolate, {kind, sig, expected_arity, suspend});
cache->PublishCounterUpdates(isolate);
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_WasmTriggerTierUp) {
SealHandleScope shs(isolate);
{
// The main code does not allocate, but interrupt handling below could.
DisallowGarbageCollection no_gc;
DCHECK_EQ(1, args.length());
Tagged<WasmTrustedInstanceData> trusted_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
FrameFinder<WasmFrame> frame_finder(isolate);
WasmFrame* frame = frame_finder.frame();
int func_index = frame->GetInnermostFunctionIndex();
DCHECK_EQ(trusted_data, frame->trusted_instance_data());
if (V8_UNLIKELY(v8_flags.wasm_sync_tier_up &&
!v8_flags.wasm_generate_compilation_hints &&
!v8_flags.trace_wasm_generate_compilation_hints)) {
if (!trusted_data->native_module()->HasCodeWithTier(
func_index, wasm::ExecutionTier::kTurbofan)) {
wasm::TierUpNowForTesting(isolate, trusted_data, func_index);
}
// We call this function when the tiering budget runs out, so reset that
// budget to appropriately delay the next call.
int array_index =
wasm::declared_function_index(trusted_data->module(), func_index);
trusted_data->tiering_budget_array()[array_index].store(
v8_flags.wasm_tiering_budget, std::memory_order_relaxed);
} else if (V8_UNLIKELY(v8_flags.wasm_generate_compilation_hints ||
v8_flags.trace_wasm_generate_compilation_hints)) {
// In this case, we do not tierup functions. However we have to remember
// that we marked them for tierup.
base::MutexGuard marked_for_tierup_mutex_guard(
&trusted_data->module()->compilation_hints_mutex);
trusted_data->module()->marked_for_tierup.emplace(func_index);
// We call this function when the tiering budget runs out, so reset that
// budget to appropriately delay the next call.
int array_index =
wasm::declared_function_index(trusted_data->module(), func_index);
trusted_data->tiering_budget_array()[array_index].store(
v8_flags.wasm_tiering_budget, std::memory_order_relaxed);
} else {
wasm::TriggerTierUp(isolate, trusted_data, func_index);
}
}
// We're reusing this interrupt mechanism to interrupt long-running loops.
StackLimitCheck check(isolate);
// We don't need to handle stack overflows here, because the function that
// performed this runtime call did its own stack check at its beginning.
// However, we can't DCHECK(!check.JsHasOverflowed()) here, because the
// additional stack space used by the CEntryStub and this runtime function
// itself might have pushed us above the limit where a stack check would
// fail.
if (check.InterruptRequested()) {
// Note: This might trigger a GC, which invalidates the {args} object (see
// https://crbug.com/v8/13036#2).
Tagged<Object> result = isolate->stack_guard()->HandleInterrupts();
if (IsExceptionHole(result)) return result;
}
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_WasmI32AtomicWait) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
int memory_index = args.smi_value_at(1);
double offset_double = args.number_value_at(2);
uintptr_t offset = static_cast<uintptr_t>(offset_double);
int32_t expected_value = NumberToInt32(args[3]);
Tagged<BigInt> timeout_ns = Cast<BigInt>(args[4]);
CppGCManaged<BackingStore>::Ptr backing_store =
trusted_instance_data->memory_object(memory_index)->backing_store();
// Should have trapped if address was OOB.
DCHECK_LT(offset, backing_store->byte_length());
// Trap if memory is not shared, or wait is not allowed on the isolate
if (!backing_store->is_shared() || !isolate->allow_atomics_wait()) {
return ThrowWasmError(
isolate, MessageTemplate::kAtomicsOperationNotAllowed,
{isolate->factory()->NewStringFromAsciiChecked("Atomics.wait")});
}
return FutexEmulation::WaitWasm32(isolate, backing_store.raw(), offset,
expected_value, timeout_ns->AsInt64());
}
RUNTIME_FUNCTION(Runtime_WasmI64AtomicWait) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
int memory_index = args.smi_value_at(1);
double offset_double = args.number_value_at(2);
uintptr_t offset = static_cast<uintptr_t>(offset_double);
Tagged<BigInt> expected_value = Cast<BigInt>(args[3]);
Tagged<BigInt> timeout_ns = Cast<BigInt>(args[4]);
CppGCManaged<BackingStore>::Ptr backing_store =
trusted_instance_data->memory_object(memory_index)->backing_store();
// Should have trapped if address was OOB.
DCHECK_LT(offset, backing_store->byte_length());
// Trap if memory is not shared, or wait is not allowed on the isolate
if (!backing_store->is_shared() || !isolate->allow_atomics_wait()) {
return ThrowWasmError(
isolate, MessageTemplate::kAtomicsOperationNotAllowed,
{isolate->factory()->NewStringFromAsciiChecked("Atomics.wait")});
}
return FutexEmulation::WaitWasm64(isolate, backing_store.raw(), offset,
expected_value->AsInt64(),
timeout_ns->AsInt64());
}
namespace {
template <typename T>
Tagged<Object> WasmManagedObjectWait(Isolate* isolate,
Tagged<HeapObject> object,
int field_offset, T expected_value,
Tagged<HeapObject> waitqueue,
Tagged<BigInt> timeout_ns) {
TSAN_ACQUIRE(object.address());
TSAN_ACQUIRE(waitqueue.address());
if (!v8_flags.wasm_skip_null_checks &&
(object == ReadOnlyRoots(isolate).wasm_null() ||
waitqueue == ReadOnlyRoots(isolate).wasm_null())) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapNullDereference);
}
if (!HeapLayout::InAnySharedSpace(object) || !isolate->allow_atomics_wait()) {
const char* op_name = IsWasmStruct(object) ? "struct.wait" : "array.wait";
return ThrowWasmError(
isolate, MessageTemplate::kAtomicsOperationNotAllowed,
{isolate->factory()->NewStringFromAsciiChecked(op_name)});
}
return FutexEmulation::WaitWasmManagedObject<T>(
isolate, object, field_offset,
Cast<Managed<FutexManagedObjectWaitList>>(waitqueue), expected_value,
timeout_ns->AsInt64());
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmManagedObjectWait32) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
Tagged<HeapObject> object = Cast<HeapObject>(args[0]);
int field_offset = args.smi_value_at(1);
int32_t expected_value = static_cast<int32_t>(args.number_value_at(2));
Tagged<HeapObject> waitqueue = Cast<HeapObject>(args[3]);
Tagged<BigInt> timeout_ns = Cast<BigInt>(args[4]);
return WasmManagedObjectWait<int32_t>(isolate, object, field_offset,
expected_value, waitqueue, timeout_ns);
}
RUNTIME_FUNCTION(Runtime_WasmManagedObjectWait64) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
Tagged<HeapObject> object = Cast<HeapObject>(args[0]);
int field_offset = args.smi_value_at(1);
Tagged<BigInt> expected_value = Cast<BigInt>(args[2]);
Tagged<HeapObject> waitqueue = Cast<HeapObject>(args[3]);
Tagged<BigInt> timeout_ns = Cast<BigInt>(args[4]);
return WasmManagedObjectWait<int64_t>(isolate, object, field_offset,
expected_value->AsInt64(), waitqueue,
timeout_ns);
}
RUNTIME_FUNCTION(Runtime_WasmManagedObjectWaitRef) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
Tagged<HeapObject> object = Cast<HeapObject>(args[0]);
int field_offset = args.smi_value_at(1);
Tagged<Object> expected_value = args[2];
Tagged<HeapObject> waitqueue = Cast<HeapObject>(args[3]);
Tagged<BigInt> timeout_ns = Cast<BigInt>(args[4]);
return WasmManagedObjectWait<Tagged<Object>>(
isolate, object, field_offset, expected_value, waitqueue, timeout_ns);
}
RUNTIME_FUNCTION(Runtime_WasmWaitqueueNew) {
HandleScope scope(isolate);
DCHECK_EQ(0, args.length());
// Memory fence is implemented in Managed<>::From.
auto ptr = std::make_shared<FutexManagedObjectWaitList>();
DirectHandle<Managed<FutexManagedObjectWaitList>> managed =
Managed<FutexManagedObjectWaitList>::From(
isolate, sizeof(FutexManagedObjectWaitList), ptr,
AllocationType::kSharedOld);
return *managed;
}
namespace {
Tagged<Object> ThrowTableOutOfBounds(
Isolate* isolate,
DirectHandle<WasmTrustedInstanceData> trusted_instance_data) {
// Handle out-of-bounds access here in the runtime call, rather
// than having the lower-level layers deal with JS exceptions.
if (isolate->context().is_null()) {
isolate->set_context(trusted_instance_data->native_context());
}
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmRefFunc) {
HandleScope scope(isolate);
DCHECK_EQ(2, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t function_index = args.positive_smi_value_at(1);
return *WasmTrustedInstanceData::GetOrCreateFuncRef(
isolate, trusted_instance_data, function_index);
}
RUNTIME_FUNCTION(Runtime_WasmInternalFunctionCreateExternal) {
HandleScope scope(isolate);
DCHECK_EQ(1, args.length());
// TODO(14564): Pass WasmFuncRef here instead of WasmInternalFunction.
DirectHandle<WasmInternalFunction> internal(
TrustedCast<WasmInternalFunction>(args[0]), isolate);
return *WasmInternalFunction::GetOrCreateExternal(internal);
}
RUNTIME_FUNCTION(Runtime_WasmFunctionTableGet) {
HandleScope scope(isolate);
DCHECK_EQ(3, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t table_index = args.positive_smi_value_at(1);
uint32_t entry_index = args.positive_smi_value_at(2);
DCHECK_LT(table_index, trusted_instance_data->tables()->ulength().value());
auto table = direct_handle(
Cast<WasmTableObject>(trusted_instance_data->tables()->get(table_index)),
isolate);
// We only use the runtime call for lazily initialized function references.
DCHECK(table->unsafe_type().ref_type_kind() == wasm::RefTypeKind::kFunction);
if (!table->is_in_bounds(entry_index)) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);
}
return *WasmTableObject::Get(isolate, table, entry_index);
}
RUNTIME_FUNCTION(Runtime_WasmFunctionTableSet) {
HandleScope scope(isolate);
DCHECK_EQ(4, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t table_index = args.positive_smi_value_at(1);
uint32_t entry_index = args.positive_smi_value_at(2);
DirectHandle<Object> element(args[3], isolate);
DCHECK_LT(table_index, trusted_instance_data->tables()->ulength().value());
auto table = direct_handle(
Cast<WasmTableObject>(trusted_instance_data->tables()->get(table_index)),
isolate);
// We only use the runtime call for lazily initialized function references.
DCHECK(table->unsafe_type().ref_type_kind() == wasm::RefTypeKind::kFunction);
if (!table->is_in_bounds(entry_index)) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);
}
DirectHandle<WasmDispatchTable> dispatch_table(
trusted_instance_data->dispatch_table(table_index), isolate);
WasmTableObject::Set(isolate, table, dispatch_table, entry_index, element);
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_WasmTableInit) {
HandleScope scope(isolate);
DCHECK_EQ(6, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t table_index = args.positive_smi_value_at(1);
uint32_t elem_segment_index = args.positive_smi_value_at(2);
static_assert(
wasm::kV8MaxWasmTableSize < kSmiMaxValue,
"Make sure clamping to Smi range doesn't make an invalid call valid");
uint32_t dst = args.positive_smi_value_at(3);
uint32_t src = args.positive_smi_value_at(4);
uint32_t count = args.positive_smi_value_at(5);
DCHECK(!isolate->context().is_null());
std::optional<MessageTemplate> opt_error =
WasmTrustedInstanceData::InitTableEntries(isolate, trusted_instance_data,
table_index, elem_segment_index,
dst, src, count);
if (opt_error.has_value()) {
return ThrowWasmError(isolate, opt_error.value());
}
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_WasmTableCopy) {
HandleScope scope(isolate);
DCHECK_EQ(6, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t table_dst_index = args.positive_smi_value_at(1);
uint32_t table_src_index = args.positive_smi_value_at(2);
static_assert(
wasm::kV8MaxWasmTableSize < kSmiMaxValue,
"Make sure clamping to Smi range doesn't make an invalid call valid");
uint32_t dst = args.positive_smi_value_at(3);
uint32_t src = args.positive_smi_value_at(4);
uint32_t count = args.positive_smi_value_at(5);
DCHECK(!isolate->context().is_null());
bool oob = !WasmTrustedInstanceData::CopyTableEntries(
isolate, trusted_instance_data, table_dst_index, table_src_index, dst,
src, count);
if (oob) return ThrowTableOutOfBounds(isolate, trusted_instance_data);
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_WasmTableGrow) {
DCHECK(isolate->IsOnCentralStack());
HandleScope scope(isolate);
DCHECK_EQ(4, args.length());
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t table_index = args.positive_smi_value_at(1);
DirectHandle<Object> value(args[2], isolate);
uint32_t delta = args.positive_smi_value_at(3);
DirectHandle<WasmTableObject> table(
Cast<WasmTableObject>(trusted_instance_data->tables()->get(table_index)),
isolate);
DirectHandle<WasmDispatchTable> dispatch_table(
trusted_instance_data->dispatch_table(table_index), isolate);
int result =
WasmTableObject::Grow(isolate, table, dispatch_table, delta, value);
return Smi::FromInt(result);
}
RUNTIME_FUNCTION(Runtime_WasmTableFill) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t table_index = args.positive_smi_value_at(1);
uint32_t start = args.positive_smi_value_at(2);
DirectHandle<Object> value(args[3], isolate);
uint32_t count = args.positive_smi_value_at(4);
DirectHandle<WasmTableObject> table(
Cast<WasmTableObject>(trusted_instance_data->tables()->get(table_index)),
isolate);
uint32_t table_size = table->current_length();
if (start > table_size) {
return ThrowTableOutOfBounds(isolate, trusted_instance_data);
}
// Even when table.fill goes out-of-bounds, as many entries as possible are
// put into the table. Only afterwards we trap.
uint32_t fill_count = std::min(count, table_size - start);
if (fill_count < count) {
return ThrowTableOutOfBounds(isolate, trusted_instance_data);
}
DirectHandle<WasmDispatchTable> dispatch_table(
trusted_instance_data->dispatch_table(table_index), isolate);
WasmTableObject::Fill(isolate, table, dispatch_table, start, value,
fill_count);
return ReadOnlyRoots(isolate).undefined_value();
}
namespace {
// Returns true if any breakpoint was hit, false otherwise.
bool ExecuteWasmDebugBreaks(
Isolate* isolate,
DirectHandle<WasmTrustedInstanceData> trusted_instance_data,
WasmFrame* frame) {
DirectHandle<Script> script{trusted_instance_data->module_object()->script(),
isolate};
auto* debug_info = trusted_instance_data->native_module()->GetDebugInfo();
// Enter the debugger.
DebugScope debug_scope(isolate->debug());
// Check for instrumentation breakpoints first, but still execute regular
// breakpoints afterwards.
bool paused_on_instrumentation = false;
DCHECK_EQ(script->break_on_entry(),
!!trusted_instance_data->break_on_entry());
if (script->break_on_entry()) {
MaybeDirectHandle<FixedArray> maybe_on_entry_breakpoints =
WasmScript::CheckBreakPoints(isolate, script,
WasmScript::kOnEntryBreakpointPosition,
frame->id());
script->set_break_on_entry(false);
// Update the "break_on_entry" flag on all live instances.
i::Tagged<i::WeakArrayList> weak_instance_list =
script->wasm_weak_instance_list();
const uint32_t weak_instance_len = weak_instance_list->length().value();
for (uint32_t i = 0; i < weak_instance_len; ++i) {
if (weak_instance_list->Get(i).IsCleared()) continue;
i::Cast<i::WasmInstanceObject>(weak_instance_list->Get(i).GetHeapObject())
->trusted_data(isolate)
->set_break_on_entry(false);
}
DCHECK(!trusted_instance_data->break_on_entry());
if (!maybe_on_entry_breakpoints.is_null()) {
isolate->debug()->OnInstrumentationBreak();
paused_on_instrumentation = true;
}
}
if (debug_info->IsStepping(frame) && !debug_info->IsFrameBlackboxed(frame)) {
debug_info->ClearStepping(isolate);
StepAction step_action = isolate->debug()->last_step_action();
isolate->debug()->ClearStepping();
isolate->debug()->OnDebugBreak(isolate->factory()->empty_fixed_array(),
step_action);
return true;
}
// Check whether we hit a breakpoint.
DirectHandle<FixedArray> breakpoints;
if (WasmScript::CheckBreakPoints(isolate, script, frame->position(),
frame->id())
.ToHandle(&breakpoints)) {
debug_info->ClearStepping(isolate);
StepAction step_action = isolate->debug()->last_step_action();
isolate->debug()->ClearStepping();
if (isolate->debug()->break_points_active()) {
// We hit one or several breakpoints. Notify the debug listeners.
isolate->debug()->OnDebugBreak(breakpoints, step_action);
}
return true;
}
return paused_on_instrumentation;
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmDebugBreak) {
HandleScope scope(isolate);
DCHECK_EQ(0, args.length());
FrameFinder<WasmFrame> frame_finder(
isolate, {StackFrame::EXIT, StackFrame::WASM_DEBUG_BREAK});
WasmFrame* frame = frame_finder.frame();
DirectHandle<WasmTrustedInstanceData> trusted_data{
frame->trusted_instance_data(), isolate};
isolate->set_context(trusted_data->native_context());
if (!ExecuteWasmDebugBreaks(isolate, trusted_data, frame)) {
// We did not hit a breakpoint. If we are in stepping code, but the user did
// not request stepping, clear this (to save further calls into this runtime
// function).
auto* debug_info = trusted_data->native_module()->GetDebugInfo();
debug_info->ClearStepping(frame);
}
// Execute a stack check before leaving this function. This is to handle any
// interrupts set by the debugger (e.g. termination), but also to execute Wasm
// code GC to get rid of temporarily created Wasm code.
StackLimitCheck check(isolate);
if (check.InterruptRequested()) {
Tagged<Object> interrupt_object =
isolate->stack_guard()->HandleInterrupts();
// Interrupt handling can create an exception, including the
// termination exception.
if (IsExceptionHole(interrupt_object)) return interrupt_object;
DCHECK(IsUndefined(interrupt_object));
}
return ReadOnlyRoots(isolate).undefined_value();
}
namespace {
DISABLE_TSAN Tagged<Object> WasmArrayCopyImpl(
Isolate* isolate, Tagged<WasmArray> dst_array, uint32_t dst_index,
Tagged<WasmArray> src_array, uint32_t src_index, uint32_t length) {
DCHECK_GT(length, 0);
bool overlapping_ranges =
dst_array.ptr() == src_array.ptr() &&
(dst_index < src_index ? dst_index + length > src_index
: src_index + length > dst_index);
wasm::CanonicalValueType element_type =
src_array->map()->wasm_type_info()->element_type();
if (element_type.is_ref()) {
ObjectSlot dst_slot = dst_array->ElementSlot(dst_index);
ObjectSlot src_slot = src_array->ElementSlot(src_index);
if (overlapping_ranges) {
isolate->heap()->MoveRange(dst_array, dst_slot, src_slot, length,
UPDATE_WRITE_BARRIER);
} else {
isolate->heap()->CopyRange(dst_array, dst_slot, src_slot, length,
UPDATE_WRITE_BARRIER);
}
} else {
void* dst = reinterpret_cast<void*>(dst_array->ElementAddress(dst_index));
void* src = reinterpret_cast<void*>(src_array->ElementAddress(src_index));
size_t copy_size = length * element_type.value_kind_size();
if (overlapping_ranges) {
MemMove(dst, src, copy_size);
} else {
MemCopy(dst, src, copy_size);
}
}
return ReadOnlyRoots(isolate).undefined_value();
}
} // namespace
// Assumes copy ranges are in-bounds and copy length > 0.
// TODO(manoskouk): Unify part of this with the implementation in
// wasm-extern-refs.cc
RUNTIME_FUNCTION(Runtime_WasmArrayCopy) {
SealHandleScope shs(isolate);
DisallowGarbageCollection no_gc;
DCHECK_EQ(5, args.length());
Tagged<WasmArray> dst_array = Cast<WasmArray>(args[0]);
uint32_t dst_index = args.positive_smi_value_at(1);
Tagged<WasmArray> src_array = Cast<WasmArray>(args[2]);
uint32_t src_index = args.positive_smi_value_at(3);
uint32_t length = args.positive_smi_value_at(4);
return WasmArrayCopyImpl(isolate, dst_array, dst_index, src_array, src_index,
length);
}
RUNTIME_FUNCTION(Runtime_WasmAllocateDescriptorStruct) {
HandleScope scope(isolate);
DCHECK_EQ(4, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_data{
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate};
DirectHandle<Map> map{Cast<Map>(args[1]), isolate};
wasm::ModuleTypeIndex type_index{args.positive_smi_value_at(2)};
DirectHandle<Object> first_field{args[3], isolate};
return *WasmStruct::AllocateDescriptorUninitialized(
isolate, trusted_data, type_index, map, first_field);
}
RUNTIME_FUNCTION(Runtime_WasmArrayNewSegment) {
HandleScope scope(isolate);
DCHECK_EQ(5, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t segment_index = args.positive_smi_value_at(1);
uint32_t offset = args.positive_smi_value_at(2);
uint32_t length = args.positive_smi_value_at(3);
DirectHandle<Map> rtt(Cast<Map>(args[4]), isolate);
Tagged<WasmTypeInfo> type_info = rtt->wasm_type_info();
wasm::CanonicalValueType element_type = type_info->element_type();
AllocationType allocation = type_info->type().is_shared()
? AllocationType::kSharedOld
: AllocationType::kYoung;
uint32_t element_size = element_type.value_kind_size();
// This check also implies no overflow.
if (length > static_cast<uint32_t>(WasmArray::MaxLength(element_size))) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapArrayTooLarge);
}
if (element_type.is_numeric()) {
// No chance of overflow due to the check above.
uint32_t length_in_bytes = length * element_size;
wasm::WireBytesRef segment_source =
trusted_instance_data->data_segments()->get(segment_index);
if (!base::IsInBounds<uint32_t>(offset, length_in_bytes,
segment_source.length())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapDataSegmentOutOfBounds);
}
base::Vector<const uint8_t> source =
trusted_instance_data->native_module()->wire_bytes() +
segment_source.offset() + offset;
return *isolate->factory()->NewWasmArrayFromMemory(length, rtt, allocation,
element_type, source);
} else {
DirectHandle<Object> elem_segment_raw(
trusted_instance_data->element_segments()->get(segment_index), isolate);
const wasm::WasmElemSegment* module_elem_segment =
&trusted_instance_data->module()->elem_segments[segment_index];
// If the segment is initialized in the instance, we have to get its length
// from there, as it might have been dropped. If the segment is
// uninitialized, we need to fetch its length from the module.
uint32_t segment_length =
IsFixedArray(*elem_segment_raw)
? Cast<FixedArray>(elem_segment_raw)->ulength().value()
: module_elem_segment->element_count;
if (!base::IsInBounds<size_t>(offset, length, segment_length)) {
return ThrowWasmError(
isolate, MessageTemplate::kWasmTrapElementSegmentOutOfBounds);
}
DirectHandle<Object> result =
isolate->factory()->NewWasmArrayFromElementSegment(
trusted_instance_data, segment_index, offset, length, rtt,
allocation, element_type);
if (IsSmi(*result)) {
return ThrowWasmError(
isolate, static_cast<MessageTemplate>(Cast<Smi>(*result).value()));
} else {
return *result;
}
}
}
namespace {
DISABLE_TSAN Tagged<Object> WasmArrayInitSegmentImpl(
Isolate* isolate,
DirectHandle<WasmTrustedInstanceData> trusted_instance_data,
uint32_t segment_index, DirectHandle<WasmArray> array, uint32_t array_index,
uint32_t segment_offset, uint32_t length) {
wasm::CanonicalValueType element_type =
array->map()->wasm_type_info()->element_type();
if (element_type.is_numeric()) {
if (!base::IsInBounds<uint32_t>(array_index, length, array->length())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapArrayOutOfBounds);
}
// No chance of overflow, due to the check above and the limit in array
// length.
uint32_t length_in_bytes = length * element_type.value_kind_size();
wasm::WireBytesRef segment_source =
trusted_instance_data->data_segments()->get(segment_index);
if (!base::IsInBounds<uint32_t>(segment_offset, length_in_bytes,
segment_source.length())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapDataSegmentOutOfBounds);
}
base::Vector<const uint8_t> wire_bytes =
trusted_instance_data->native_module()->wire_bytes();
const uint8_t* source =
wire_bytes.begin() + segment_source.offset() + segment_offset;
void* dest = reinterpret_cast<void*>(array->ElementAddress(array_index));
#if V8_TARGET_BIG_ENDIAN
MemCopyAndSwitchEndianness(dest, source, length,
element_type.value_kind_size());
#else
MemCopy(dest, source, length_in_bytes);
#endif
return *isolate->factory()->undefined_value();
} else {
DirectHandle<Object> elem_segment_raw(
trusted_instance_data->element_segments()->get(segment_index), isolate);
const wasm::WasmElemSegment* module_elem_segment =
&trusted_instance_data->module()->elem_segments[segment_index];
// If the segment is initialized in the instance, we have to get its length
// from there, as it might have been dropped. If the segment is
// uninitialized, we need to fetch its length from the module.
uint32_t segment_length =
IsFixedArray(*elem_segment_raw)
? Cast<FixedArray>(elem_segment_raw)->ulength().value()
: module_elem_segment->element_count;
if (!base::IsInBounds<size_t>(segment_offset, length, segment_length)) {
return ThrowWasmError(
isolate, MessageTemplate::kWasmTrapElementSegmentOutOfBounds);
}
if (!base::IsInBounds(array_index, length, array->length())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapArrayOutOfBounds);
}
// If the element segment has not been initialized yet, lazily initialize it
// now.
std::optional<MessageTemplate> opt_error = wasm::InitializeElementSegment(
isolate, trusted_instance_data, segment_index);
if (opt_error.has_value()) {
return ThrowWasmError(isolate, opt_error.value());
}
auto elements = direct_handle(
Cast<FixedArray>(
trusted_instance_data->element_segments()->get(segment_index)),
isolate);
if (length > 0) {
isolate->heap()->CopyRange(*array, array->ElementSlot(array_index),
elements->RawFieldOfElementAt(segment_offset),
length, UPDATE_WRITE_BARRIER);
}
return *isolate->factory()->undefined_value();
}
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmArrayInitSegment) {
HandleScope scope(isolate);
DCHECK_EQ(6, args.length());
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t segment_index = args.positive_smi_value_at(1);
DirectHandle<WasmArray> array(Cast<WasmArray>(args[2]), isolate);
uint32_t array_index = args.positive_smi_value_at(3);
uint32_t segment_offset = args.positive_smi_value_at(4);
uint32_t length = args.positive_smi_value_at(5);
return WasmArrayInitSegmentImpl(isolate, trusted_instance_data, segment_index,
array, array_index, segment_offset, length);
}
// Allocate a new suspender, and prepare for stack switching by updating the
// active continuation, active suspender and stack limit.
RUNTIME_FUNCTION(Runtime_WasmAllocateSuspender) {
HandleScope scope(isolate);
DirectHandle<WasmSuspenderObject> suspender =
isolate->factory()->NewWasmSuspenderObjectInitialized();
// Update the stack state.
std::unique_ptr<wasm::StackMemory> target_stack =
isolate->stack_pool().GetOrAllocate();
target_stack->jmpbuf()->parent = nullptr;
target_stack->jmpbuf()->stack_limit = target_stack->jslimit();
target_stack->jmpbuf()->sp = target_stack->base();
target_stack->jmpbuf()->fp = kNullAddress;
target_stack->jmpbuf()->state = wasm::JumpBuffer::Suspended;
target_stack->jmpbuf()->is_on_central_stack = false;
if (v8_flags.wasm_wasmfx) {
// For now JSPI does not use the WasmStackObject, and it is only set here
// because it is expected by WasmFX.
// TODO(thibaudm): We could consider using this object for JSPI too as an
// indirection between the WasmSuspenderObjects and the StackMemory. This
// would have roughly the same benefits as for WasmFX:
// - We would only need to allocate and manage a single EPT entry per
// StackMemory,
// - It would be easier to track ownership of the StackMemory and ensure
// that there is no UAF. In particular the StackMemory could track its
// (unique) EPT entry via {EPT::ManagedResource} and zap it when the
// resource is freed.
target_stack->set_stack_obj(
*isolate->factory()->NewWasmStackObject(target_stack.get()));
}
// Update the suspender state.
Tagged<WasmSuspenderObject> active_suspender =
isolate->isolate_data()->active_suspender();
if (v8_flags.wasm_wasmfx) {
// The active suspender is about to become inactive. Record the currently
// active stack (which may have changed due to WasmFX) for when we
// return to this suspender.
active_suspender->set_stack(isolate,
isolate->isolate_data()->active_stack());
}
suspender->set_parent(active_suspender);
suspender->set_stack(isolate, target_stack.get());
// The active stack is updated in {Isolate::SwitchStacks}.
isolate->isolate_data()->set_active_suspender(*suspender);
target_stack->set_index(isolate->wasm_stacks().size());
isolate->wasm_stacks().emplace_back(std::move(target_stack));
for (size_t i = 0; i < isolate->wasm_stacks().size(); ++i) {
SLOW_DCHECK(isolate->wasm_stacks()[i]->index() == i);
}
// Stack limit will be updated in WasmReturnPromiseOnSuspendAsm builtin.
return *suspender;
}
namespace {
int GetWasmFrameCount(Isolate* isolate, Tagged<WasmSuspenderObject> suspender) {
int count = 0;
for (StackFrameIterator it(isolate); !it.done(); it.Advance()) {
StackFrame* frame = it.frame();
#ifdef DEBUG
Tagged<WasmSuspenderObject> parent = suspender->parent();
wasm::StackMemory* end_stack = parent->stack();
DCHECK_NOT_NULL(end_stack);
bool suspender_contains_frame = false;
for (wasm::StackMemory* stack = isolate->isolate_data()->active_stack();
stack != end_stack; stack = stack->jmpbuf()->parent) {
if (stack->Contains(frame->fp())) {
suspender_contains_frame = true;
break;
}
}
#endif
if (frame->is_wasm()) {
WasmFrame* wasm_frame = WasmFrame::cast(frame);
count += wasm_frame->Summarize().size();
DCHECK(suspender_contains_frame);
} else if (frame->is_javascript()) {
// By construction, the first JS frame must be the JSPI entry point and is
// outside of the captured stack. Stop the count.
DCHECK(!suspender_contains_frame);
break;
}
}
return count;
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmSuspended) {
HandleScope scope(isolate);
DirectHandle<JSPromise> awaited_promise(Cast<JSPromise>(args[0]), isolate);
DirectHandle<WasmSuspenderObject> suspender(
TrustedCast<WasmSuspenderObject>(args[1]), isolate);
DirectHandle<JSPromise> throwaway =
isolate->factory()->NewJSPromiseWithoutHook();
int skip_frame_count = GetWasmFrameCount(isolate, *suspender);
isolate->OnAsyncFunctionSuspended(throwaway, awaited_promise,
skip_frame_count);
throwaway->set_has_handler(true);
if (isolate->debug()->is_active()) {
Tagged<Object> promise_obj = suspender->promise();
if (IsJSPromise(promise_obj)) {
DirectHandle<JSPromise> outer_promise(Cast<JSPromise>(promise_obj),
isolate);
Object::SetProperty(isolate, throwaway,
isolate->factory()->promise_handled_by_symbol(),
outer_promise, StoreOrigin::kMaybeKeyed,
Just(ShouldThrow::kThrowOnError))
.Check();
}
}
return *throwaway;
}
// Helper function needed for the stress stack switching mode.
// This is a runtime function to avoid writing trusted space memory from
// generated code.
RUNTIME_FUNCTION(Runtime_ClearWasmSuspenderResumeField) {
SealHandleScope shs(isolate);
DisallowGarbageCollection no_gc;
// Should only be used in stress stack switching mode.
CHECK(v8_flags.stress_wasm_stack_switching);
DCHECK_EQ(1, args.length());
Tagged<WasmSuspenderObject> suspender =
TrustedCast<WasmSuspenderObject>(args[0]);
suspender->set_resume(ReadOnlyRoots(isolate).undefined_value());
return ReadOnlyRoots(isolate).undefined_value();
}
namespace {
class PrototypesSetup : public wasm::Decoder {
public:
struct Method {
enum Kind : uint8_t { kMethod = 0, kGetter = 1, kSetter = 2 };
Kind kind;
bool is_static;
base::Vector<const uint8_t> name;
};
PrototypesSetup(Isolate* isolate, base::Vector<const uint8_t> data)
: Decoder(data), isolate_(isolate) {
// kLength == 0 may look weird, but it's what you'd get for
// function wrapper(...args) { return wasm_func(this, ...args); }.
static constexpr int kLength = 0;
method_wrapper_ = isolate_->factory()->NewSharedFunctionInfoForBuiltin(
{}, Builtin::kWasmMethodWrapper, kLength, kDontAdapt);
method_wrapper_->set_native(true);
method_wrapper_->set_language_mode(LanguageMode::kStrict);
}
MaybeDirectHandle<String> ReadUtf8String(base::Vector<const uint8_t> bytes) {
DirectHandle<String> result;
if (!isolate()
->factory()
->NewStringFromUtf8(bytes, unibrow::Utf8Variant::kUtf8)
.ToHandle(&result)) {
DCHECK(isolate()->has_exception());
return {};
}
return isolate()->factory()->InternalizeString(result);
}
Tagged<Object> SetupPrototypes(DirectHandle<Object> constructors) {
uint32_t num_prototypes = consume_u32v("number of prototypes");
FOR_WITH_HANDLE_SCOPE(isolate(), uint32_t proto_index = 0, proto_index,
proto_index < num_prototypes && ok(), proto_index++) {
// We have to support {null} prototypes until we try to do something
// with them. To avoid needing casts all over the following code, we
// handle that edge case separately: {prototype.is_null()} (i.e. empty
// handle) usually means "error", *unless* {prototype_is_null = true},
// in which case it means "prototype is {null}".
bool prototype_is_null = false;
DirectHandle<JSReceiver> prototype;
if (!NextPrototype(&prototype_is_null).To(&prototype) &&
!prototype_is_null) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
bool fast_path = CheckFastPathEligibility(prototype);
uint32_t has_constructor = consume_u32v("constructor");
if (!ok()) break;
if (has_constructor == 1) {
if (V8_UNLIKELY(prototype_is_null)) {
isolate()->Throw(*isolate()->factory()->NewTypeError(
MessageTemplate::kNonObjectPropertyStoreWithProperty,
isolate()->factory()->null_value(),
isolate()->factory()->constructor_string()));
return ReadOnlyRoots(isolate()).exception();
}
DirectHandle<JSFunction> constructor;
if (!NextFunction().To(&constructor)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
uint32_t name_length = consume_u32v("constructor name length");
if (!ok()) break;
const uint8_t* name_start = pc();
consume_bytes(name_length);
if (!ok()) break;
DirectHandle<JSFunction> wrapped_constructor = InstallConstructor(
prototype, constructor, {name_start, name_length}, constructors);
if (wrapped_constructor.is_null()) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
uint32_t num_statics = consume_u32v("number of statics");
if (!ok()) break;
if (num_statics != 0) {
if (num_statics > wasm::kMaxConfigureAllStaticProperties) {
isolate()->Throw(*isolate()->factory()->NewRangeError(
MessageTemplate::kOutOfMemory,
isolate()->factory()->NewStringFromAsciiChecked(
"too many constructor properties in configureAll")));
return ReadOnlyRoots(isolate()).exception();
}
ToDictionaryMode(wrapped_constructor, num_statics);
for (uint32_t i = 0; i < num_statics; i++) {
Method method = NextMethod(true);
if (!ok()) break;
DirectHandle<JSFunction> function;
if (!NextFunction().To(&function) ||
!InstallMethod(wrapped_constructor, method, function)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
}
if (!ok()) break;
}
} else if (has_constructor > 1) {
// Contrary to other uses of the Decoder, the built-in offset reporting
// is not usable here, so we have to hand-roll it.
errorf(0u, "invalid constructor count %d at data+%u", has_constructor,
pc_offset() - 1);
break;
}
uint32_t num_methods = consume_u32v("number of methods");
if (!ok()) break;
if (num_methods > wasm::kMaxConfigureAllMethods) {
isolate()->Throw(*isolate()->factory()->NewRangeError(
MessageTemplate::kOutOfMemory,
isolate()->factory()->NewStringFromAsciiChecked(
"too many methods in configureAll")));
return ReadOnlyRoots(isolate()).exception();
}
DirectHandle<NameDictionary> dictionary;
if (!prototype_is_null) {
if (IsJSObject(*prototype)) {
DirectHandle<JSObject> proto = Cast<JSObject>(prototype);
ToDictionaryMode(proto, num_methods);
if (fast_path) {
dictionary = handle(proto->property_dictionary(), isolate_);
}
}
} else if (num_methods > 0) {
DCHECK(prototype_is_null);
isolate()->Throw(*isolate()->factory()->NewTypeError(
MessageTemplate::kNonObjectPropertyStoreWithProperty,
isolate()->factory()->null_value(),
isolate()->factory()->constructor_string()));
return ReadOnlyRoots(isolate()).exception();
}
base::Vector<const uint8_t> last_name;
DirectHandle<JSFunction> getter;
DirectHandle<JSFunction> setter;
for (uint32_t i = 0; i < num_methods; i++) {
Method method = NextMethod(false);
if (!ok()) break;
DirectHandle<JSFunction> function;
if (!NextFunction().To(&function)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
// Wrap the function to pass the JS receiver as first Wasm param.
DirectHandle<Context> context = isolate_->factory()->NewBuiltinContext(
isolate_->native_context(), wasm::kMethodWrapperContextLength);
context->SetNoCell(wasm::kMethodWrapperContextSlot, *function);
function =
Factory::JSFunctionBuilder{isolate_, method_wrapper(), context}
.set_map(
isolate_->strict_function_with_readonly_prototype_map())
.Build();
// For nicer stack traces, we could call
// JSFunction::SetName(..., function, name, ...);
// here, but that would have a performance cost, so for now we choose
// to hide the wrappers on stack traces instead.
// Note to our future selves: if we wanted names, the most performant
// way would likely be to add a pre-created
// strict_function_with_NAME_AND_readonly_prototype_map
// and use that for these wrappers, to avoid needing individual SFIs
// for storing their names.
if (fast_path) {
bool bailout = MayBeArrayIndex(method.name);
// If we have pending accessors, and we're moving on to something
// else, install them now.
if ((!getter.is_null() || !setter.is_null()) &&
(method.name != last_name || method.kind == Method::kMethod ||
bailout)) {
if (!FastPath_AddAccessorProperty(prototype, dictionary, last_name,
getter, setter)
.ToHandle(&dictionary)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
getter = {};
setter = {};
}
if (bailout) {
fast_path = false;
prototype->SetProperties(*dictionary);
if (!InstallMethod(prototype, method, function)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
continue;
}
if (method.kind == Method::kMethod) {
if (!FastPath_AddDataProperty(prototype, dictionary, method.name,
function)
.ToHandle(&dictionary)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
} else if (method.kind == Method::kGetter) {
getter = function;
last_name = method.name;
} else if (method.kind == Method::kSetter) {
setter = function;
last_name = method.name;
}
} else {
if (!InstallMethod(prototype, method, function)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
}
}
// If we used the fast path, then the extended property dictionary
// hasn't been installed yet, and we may still have pending accessors.
if (fast_path) {
if (!getter.is_null() || !setter.is_null()) {
if (!FastPath_AddAccessorProperty(prototype, dictionary, last_name,
getter, setter)
.ToHandle(&dictionary)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
}
prototype->SetProperties(*dictionary);
}
if (!ok()) break;
uint32_t parent_idx_offset = pc_offset();
int32_t parent_idx = consume_i32v("parentidx");
if (!ok()) break;
if (parent_idx >= 0 && static_cast<uint32_t>(parent_idx) < proto_index) {
if (prototype_is_null) {
isolate()->Throw(*isolate()->factory()->NewTypeError(
MessageTemplate::kCalledOnNullOrUndefined,
isolate()->factory()->NewStringFromAsciiChecked(
"Object.setPrototypeOf")));
return ReadOnlyRoots(isolate()).exception();
}
// Previous entries in the prototypes array may have been overwritten
// since we last saw them, so we cannot make any assumptions about them.
DirectHandle<Object> parent =
PrototypeByIndex(static_cast<uint32_t>(parent_idx));
if (!JSReceiver::SetPrototype(isolate(), prototype, parent, true,
ShouldThrow::kThrowOnError)
.FromMaybe(false)) {
DCHECK(isolate()->has_exception());
return ReadOnlyRoots(isolate()).exception();
}
} else if (parent_idx == -1) {
// No parent requested.
} else {
errorf(0u, "invalid parentidx %d at data+%u", parent_idx,
parent_idx_offset);
break;
}
}
if (HasMoreFunctions()) error("unconsumed functions");
if (HasMorePrototypes()) error("unconsumed prototypes");
if (more()) error("unconsumed data");
if (!ok()) {
DirectHandle<String> message =
isolate()
->factory()
->NewStringFromUtf8(base::VectorOf(error().message()))
.ToHandleChecked();
DirectHandle<JSObject> error = isolate()->factory()->NewError(
isolate()->wasm_runtime_error_function(), message);
return isolate()->Throw(*error);
}
return ReadOnlyRoots(isolate()).undefined_value();
}
Method NextMethod(bool is_static) {
uint8_t kind = consume_u8("kind");
if (kind > 2) {
errorf(0u, "invalid method kind %u at data+%u", kind, pc_offset() - 1);
return {};
}
uint32_t name_length = consume_u32v("name length");
if (!ok()) return {};
const uint8_t* name_start = pc();
consume_bytes(name_length);
if (!ok()) return {};
return {.kind = static_cast<Method::Kind>(kind),
.is_static = is_static,
.name = {name_start, name_length}};
}
MaybeDirectHandle<JSFunction> NextFunction() {
DirectHandle<Object> maybe_func = NextFunctionInternal();
if (maybe_func.is_null()) {
ThrowWasmError(isolate_, MessageTemplate::kWasmTrapArrayOutOfBounds);
return {};
}
// TODO(jkummerow): Can we tighten the spec to require non-nullable arrays?
if (IsWasmNull(*maybe_func)) {
ThrowWasmError(isolate_, MessageTemplate::kWasmTrapNullFunc);
return {};
}
DirectHandle<WasmFuncRef> funcref = Cast<WasmFuncRef>(maybe_func);
DirectHandle<WasmInternalFunction> internal_function(
funcref->internal(isolate_), isolate_);
return Cast<JSFunction>(
WasmInternalFunction::GetOrCreateExternal(internal_function));
}
MaybeDirectHandle<JSReceiver> NextPrototype(bool* value_was_null) {
DirectHandle<Object> maybe_proto = NextPrototypeInternal();
if (maybe_proto.is_null()) {
ThrowWasmError(isolate_, MessageTemplate::kWasmTrapArrayOutOfBounds);
return {};
}
if (IsHeapObject(*maybe_proto)) {
DirectHandle<HeapObject> heap_proto = Cast<HeapObject>(maybe_proto);
if (IsJSObject(*heap_proto)) {
if (HeapLayout::InWritableSharedSpace(*heap_proto)) {
DCHECK(v8_flags.harmony_struct);
// Shared JS structs are not supported as prototypes, and probably
// never will be: we cannot change their maps to add new properties,
// and we cannot add pointers to non-shared methods to them.
ThrowWasmError(isolate_, MessageTemplate::kWasmTrapIllegalCast);
return {};
}
DirectHandle<JSObject> proto = Cast<JSObject>(heap_proto);
if (!proto->map()->is_prototype_map()) {
// Important for correctness: switch to a non-shared map.
// Important for performance: switch to dictionary mode.
JSObject::OptimizeAsPrototype(proto);
} else {
// Prepare the fast path by invalidating any ICs that might already
// rely on this prototype.
JSObject::InvalidatePrototypeChains(proto->map());
}
return proto;
}
if (IsJSReceiver(*heap_proto)) {
return Cast<JSReceiver>(heap_proto);
}
if (IsNull(*heap_proto)) {
*value_was_null = true;
return {};
}
}
isolate_->Throw(*isolate_->factory()->NewTypeError(
MessageTemplate::kProtoObjectOrNull, maybe_proto));
return {};
}
// Adding multiple properties is more efficient when the prototype
// object is in dictionary mode. ICs will transition it back to
// "fast" (but slow to modify) properties.
void ToDictionaryMode(DirectHandle<JSObject> object, int num_properties) {
if (!object->HasFastProperties()) return;
if (IsJSGlobalProxy(*object)) return;
JSObject::NormalizeProperties(isolate_, Cast<JSObject>(object),
KEEP_INOBJECT_PROPERTIES, num_properties,
"Wasm prototype setup");
}
bool InstallMethod(DirectHandle<JSReceiver> receiver, Method method,
DirectHandle<JSFunction> function) {
DirectHandle<String> name;
if (!ReadUtf8String(method.name).To(&name)) return false;
PropertyDescriptor prop;
prop.set_enumerable(false);
prop.set_configurable(true);
if (method.kind == Method::kMethod) {
prop.set_writable(true);
prop.set_value(function);
} else if (method.kind == Method::kGetter) {
prop.set_get(function);
} else if (method.kind == Method::kSetter) {
prop.set_set(function);
} else {
UNREACHABLE(); // Ruled out by validation.
}
return JSReceiver::DefineOwnProperty(isolate_, receiver, name, &prop,
Just(ShouldThrow::kThrowOnError))
.FromMaybe(false);
}
// Returns the wrapped constructor on success.
DirectHandle<JSFunction> InstallConstructor(
DirectHandle<JSReceiver> prototype,
DirectHandle<JSFunction> wasm_function,
base::Vector<const uint8_t> name_vec,
DirectHandle<Object> all_constructors) {
DirectHandle<String> name;
if (!ReadUtf8String(name_vec).To(&name)) return {};
if (!IsJSReceiver(*all_constructors)) {
isolate_->Throw(*isolate_->factory()->NewTypeError(
MessageTemplate::kNonObjectPropertyStoreWithProperty,
all_constructors, name));
return {};
}
DirectHandle<Context> context = isolate_->factory()->NewBuiltinContext(
isolate_->native_context(), wasm::kConstructorFunctionContextLength);
context->SetNoCell(wasm::kConstructorFunctionContextSlot, *wasm_function);
Builtin code = Builtin::kWasmConstructorWrapper;
const uint32_t length = wasm_function->length();
DirectHandle<SharedFunctionInfo> sfi =
isolate_->factory()->NewSharedFunctionInfoForBuiltin(
name, code, static_cast<int>(length), kDontAdapt);
sfi->set_native(true);
sfi->set_language_mode(LanguageMode::kStrict);
DirectHandle<JSFunction> constructor =
Factory::JSFunctionBuilder{isolate_, sfi, context}
.set_map(isolate_->strict_function_with_readonly_prototype_map())
.Build();
constructor->set_prototype_or_initial_map(*prototype, kReleaseStore);
PropertyDescriptor constructor_prop;
constructor_prop.set_enumerable(false);
constructor_prop.set_configurable(true);
constructor_prop.set_writable(true);
constructor_prop.set_value(constructor);
if (!JSReceiver::DefineOwnProperty(
isolate_, prototype, isolate_->factory()->constructor_string(),
&constructor_prop, Just(ShouldThrow::kThrowOnError))
.FromMaybe(false)) {
return {};
}
PropertyDescriptor prop;
prop.set_enumerable(true);
prop.set_configurable(true);
prop.set_writable(true);
prop.set_value(constructor);
if (!JSReceiver::DefineOwnProperty(isolate_,
Cast<JSReceiver>(all_constructors), name,
&prop, Just(ShouldThrow::kThrowOnError))
.FromMaybe(false)) {
return {};
}
return constructor;
}
// Fast path, skipping the LookupIterator.
bool CheckFastPathEligibility(DirectHandle<JSReceiver> prototype) {
if constexpr (V8_ENABLE_SWISS_NAME_DICTIONARY_BOOL) {
// Support for swiss name dictionaries isn't implemented here yet.
UNIMPLEMENTED();
}
if (prototype.is_null()) return false;
Tagged<Map> map = prototype->map();
if (map->instance_type() != JS_OBJECT_TYPE) return false;
if (!map->is_extensible()) return false;
// JS_OBJECT_TYPE implies no access checks or interceptors.
DCHECK(!map->is_access_check_needed());
DCHECK(!map->has_indexed_interceptor());
DCHECK(!map->has_named_interceptor());
DCHECK(!IsJSGlobalObject(*prototype));
if (map->is_dictionary_map()) {
if (Cast<JSObject>(prototype)
->property_dictionary()
->NumberOfElements() != 0) {
return false;
}
} else {
if (map->NumberOfOwnDescriptors() != 0) return false;
}
return true;
}
bool MayBeArrayIndex(base::Vector<const uint8_t> name) {
// The fast path can't handle elements (i.e. names that are string
// representations of array indices). To save time, we approximate
// detection of such names by only looking at the first character.
if (name.size() == 0) return false;
char first = name.at(0);
return first >= '0' && first <= '9';
}
// This is not fully generic: it doesn't need to handle overwriting arbitrary
// properties, only those that the same fast path put there before.
MaybeDirectHandle<NameDictionary> FastPath_AddDataProperty(
DirectHandle<JSReceiver> receiver,
DirectHandle<NameDictionary> property_dictionary,
base::Vector<const uint8_t> name_vec, DirectHandle<Object> value) {
DirectHandle<String> name;
if (!ReadUtf8String(name_vec).To(&name)) return {};
PropertyDetails details(PropertyKind::kData, DONT_ENUM,
PropertyCellType::kConstant);
InternalIndex entry = property_dictionary->FindEntry(isolate_, name);
if (entry.is_found()) [[unlikely]] {
// This isn't expected to happen in practice, so from a performance
// perspective it'd be fine to just bail out of the fast path; but the
// code ends up being simpler if we just handle this and carry on.
PropertyDetails old_details = property_dictionary->DetailsAt(entry);
details = details.set_index(old_details.dictionary_index());
details = details.set_cell_type(PropertyCellType::kMutable);
property_dictionary->DetailsAtPut(entry, details);
property_dictionary->ValueAtPut(entry, *value);
return property_dictionary;
}
DCHECK(entry.is_not_found());
if (!NameDictionary::Add(isolate_, property_dictionary, name, value,
details)
.ToHandle(&property_dictionary)) {
return {};
}
LookupIterator::UpdateProtector(isolate_, receiver, name, value);
if (name->IsInteresting(isolate_)) {
property_dictionary->set_may_have_interesting_properties(true);
}
return property_dictionary;
}
// This is not fully generic: it doesn't need to handle overwriting arbitrary
// properties, only those that the same fast path put there before.
MaybeDirectHandle<NameDictionary> FastPath_AddAccessorProperty(
DirectHandle<JSReceiver> receiver,
DirectHandle<NameDictionary> property_dictionary,
base::Vector<const uint8_t> name_vec, DirectHandle<JSFunction> getter,
DirectHandle<JSFunction> setter) {
DirectHandle<String> name;
if (!ReadUtf8String(name_vec).To(&name)) return {};
PropertyDetails details(PropertyKind::kAccessor, DONT_ENUM,
PropertyCellType::kMutable);
DirectHandle<AccessorPair> pair = isolate_->factory()->NewAccessorPair();
if (!getter.is_null()) pair->set_getter(*getter);
if (!setter.is_null()) pair->set_setter(*setter);
InternalIndex entry = property_dictionary->FindEntry(isolate_, name);
if (entry.is_found()) [[unlikely]] {
// This isn't expected to happen in practice, so from a performance
// perspective it'd be fine to just bail out of the fast path; but the
// code ends up being simpler if we just handle this and carry on.
PropertyDetails old_details = property_dictionary->DetailsAt(entry);
details = details.set_index(old_details.dictionary_index());
// When setting only one accessor, keep the other if it's already present.
if (getter.is_null() || setter.is_null()) {
Tagged<Object> old_value = property_dictionary->ValueAt(entry);
if (IsAccessorPair(old_value)) {
Tagged<AccessorPair> old_pair = Cast<AccessorPair>(old_value);
if (getter.is_null() && !IsNull(old_pair->getter())) {
pair->set_getter(old_pair->getter());
}
if (setter.is_null() && !IsNull(old_pair->setter())) {
pair->set_setter(old_pair->setter());
}
}
}
property_dictionary->DetailsAtPut(entry, details);
property_dictionary->ValueAtPut(entry, *pair);
return property_dictionary;
}
DCHECK(entry.is_not_found());
if (!NameDictionary::Add(isolate_, property_dictionary, name, pair, details)
.ToHandle(&property_dictionary)) {
return {};
}
LookupIterator::UpdateProtector(isolate_, receiver, name, pair);
if (name->IsInteresting(isolate_)) {
property_dictionary->set_may_have_interesting_properties(true);
}
return property_dictionary;
}
protected:
Isolate* isolate() { return isolate_; }
virtual DirectHandle<Object> NextFunctionInternal() = 0;
virtual bool HasMoreFunctions() = 0;
virtual DirectHandle<Object> NextPrototypeInternal() = 0;
virtual bool HasMorePrototypes() = 0;
virtual DirectHandle<Object> PrototypeByIndex(uint32_t index) = 0;
DirectHandle<SharedFunctionInfo> method_wrapper() { return method_wrapper_; }
private:
Isolate* isolate_;
DirectHandle<SharedFunctionInfo> method_wrapper_;
};
class PrototypesSetup_Arrays : public PrototypesSetup {
public:
PrototypesSetup_Arrays(Isolate* isolate, base::Vector<const uint8_t> data,
DirectHandle<WasmArray> prototypes,
DirectHandle<WasmArray> functions)
: PrototypesSetup(isolate, data),
prototypes_(prototypes),
functions_(functions) {}
protected:
DirectHandle<Object> NextFunctionInternal() override {
if (function_index_ >= functions_->length()) return {};
return WasmArray::GetElement(isolate(), functions_, function_index_++);
}
bool HasMoreFunctions() override {
return function_index_ < functions_->length();
}
DirectHandle<Object> NextPrototypeInternal() override {
if (prototype_index_ >= prototypes_->length()) return {};
return WasmArray::GetElement(isolate(), prototypes_, prototype_index_++);
}
bool HasMorePrototypes() override {
return prototype_index_ < prototypes_->length();
}
DirectHandle<Object> PrototypeByIndex(uint32_t index) override {
DCHECK_LT(index, prototypes_->length());
return WasmArray::GetElement(isolate(), prototypes_, index);
}
private:
DirectHandle<WasmArray> prototypes_;
DirectHandle<WasmArray> functions_;
uint32_t prototype_index_{0};
uint32_t function_index_{0};
};
class PrototypesSetup_Sections : public PrototypesSetup {
public:
PrototypesSetup_Sections(Isolate* isolate, base::Vector<const uint8_t> data,
DirectHandle<FixedArray> prototypes,
uint32_t prototypes_start_index,
uint32_t prototypes_length,
DirectHandle<FixedArray> functions,
uint32_t functions_start_index,
uint32_t functions_length)
: PrototypesSetup(isolate, data),
prototypes_(prototypes),
functions_(functions),
prototype_start_index_(prototypes_start_index),
prototype_index_(prototypes_start_index),
prototypes_end_(prototypes_start_index + prototypes_length),
function_index_(functions_start_index),
functions_end_(functions_start_index + functions_length) {}
protected:
DirectHandle<Object> NextFunctionInternal() override {
if (function_index_ >= functions_end_) return {};
return direct_handle(functions_->get(function_index_++), isolate());
}
bool HasMoreFunctions() override { return function_index_ < functions_end_; }
DirectHandle<Object> NextPrototypeInternal() override {
if (prototype_index_ >= prototypes_end_) return {};
return direct_handle(prototypes_->get(prototype_index_++), isolate());
}
bool HasMorePrototypes() override {
return prototype_index_ < prototypes_end_;
}
DirectHandle<Object> PrototypeByIndex(uint32_t index) override {
index += prototype_start_index_;
DCHECK_LT(index, prototypes_end_);
return direct_handle(prototypes_->get(index), isolate());
}
private:
DirectHandle<FixedArray> prototypes_;
DirectHandle<FixedArray> functions_;
uint32_t prototype_start_index_;
uint32_t prototype_index_;
uint32_t prototypes_end_;
uint32_t function_index_;
uint32_t functions_end_;
};
MaybeDirectHandle<FixedArray> GetElementSegment(
Isolate* isolate, DirectHandle<WasmTrustedInstanceData> instance,
uint32_t segment_index) {
Tagged<Object> segment_raw = instance->element_segments()->get(segment_index);
if (IsFixedArray(segment_raw)) {
return {Cast<FixedArray>(segment_raw), isolate};
}
std::optional<MessageTemplate> opt_error = wasm::InitializeElementSegment(
isolate, instance, segment_index, wasm::kPrecreateExternal);
if (opt_error.has_value()) {
ThrowWasmError(isolate, opt_error.value());
return {};
}
return {Cast<FixedArray>(instance->element_segments()->get(segment_index)),
isolate};
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmConfigureAllPrototypes) {
HandleScope scope(isolate);
DCHECK_EQ(4, args.length());
MessageTemplate illegal_cast = MessageTemplate::kWasmTrapIllegalCast;
if (!IsWasmArray(args[0])) return ThrowWasmError(isolate, illegal_cast);
if (!IsWasmArray(args[1])) return ThrowWasmError(isolate, illegal_cast);
if (!IsWasmArray(args[2])) return ThrowWasmError(isolate, illegal_cast);
DirectHandle<WasmArray> prototypes(Cast<WasmArray>(args[0]), isolate);
DirectHandle<WasmArray> functions(Cast<WasmArray>(args[1]), isolate);
DirectHandle<WasmArray> data(Cast<WasmArray>(args[2]), isolate);
DirectHandle<Object> constructors(args[3], isolate);
{
Tagged<Object> expected_prototypes_map =
MakeStrong(isolate->heap()->wasm_canonical_rtts()->get(
wasm::TypeCanonicalizer::kPredefinedArrayExternRefIndex.index));
Tagged<Object> expected_functions_map =
MakeStrong(isolate->heap()->wasm_canonical_rtts()->get(
wasm::TypeCanonicalizer::kPredefinedArrayFuncRefIndex.index));
Tagged<Object> expected_data_map =
MakeStrong(isolate->heap()->wasm_canonical_rtts()->get(
wasm::TypeCanonicalizer::kPredefinedArrayI8Index.index));
if (prototypes->map() != expected_prototypes_map ||
functions->map() != expected_functions_map ||
data->map() != expected_data_map) {
return ThrowWasmError(isolate, illegal_cast);
}
}
// Arrays on the heap can move on GC, so we create an immovable copy of
// the data we'll need to decode.
uint32_t length = data->length();
base::OwnedVector<uint8_t> immovable_data;
if (length) {
immovable_data = base::OwnedCopyOf(
reinterpret_cast<const uint8_t*>(data->ElementAddress(0)), length);
}
PrototypesSetup_Arrays decoder(isolate, immovable_data.as_vector(),
prototypes, functions);
return decoder.SetupPrototypes(constructors);
}
RUNTIME_FUNCTION(Runtime_WasmConfigureAllPrototypesOpt) {
HandleScope scope(isolate);
DCHECK_EQ(3, args.length());
uint32_t* stack_buffer = reinterpret_cast<uint32_t*>(args[0].ptr());
DirectHandle<Object> constructors(args[1], isolate);
DirectHandle<WasmTrustedInstanceData> instance(
TrustedCast<WasmTrustedInstanceData>(args[2]), isolate);
uint32_t prototypes_start = stack_buffer[0];
uint32_t prototypes_length = stack_buffer[1];
uint32_t prototypes_segment_index = stack_buffer[2];
uint32_t functions_start = stack_buffer[3];
uint32_t functions_length = stack_buffer[4];
uint32_t functions_segment_index = stack_buffer[5];
uint32_t data_start = stack_buffer[6];
uint32_t data_length = stack_buffer[7];
uint32_t data_segment_index = stack_buffer[8];
DirectHandle<FixedArray> prototypes_segment;
if (!GetElementSegment(isolate, instance, prototypes_segment_index)
.ToHandle(&prototypes_segment)) {
DCHECK(isolate->has_exception());
return ReadOnlyRoots(isolate).exception();
}
if (!base::IsInBounds<size_t>(prototypes_start, prototypes_length,
prototypes_segment->ulength().value())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapElementSegmentOutOfBounds);
}
DirectHandle<FixedArray> functions_segment;
if (!GetElementSegment(isolate, instance, functions_segment_index)
.ToHandle(&functions_segment)) {
DCHECK(isolate->has_exception());
return ReadOnlyRoots(isolate).exception();
}
if (!base::IsInBounds<size_t>(functions_start, functions_length,
functions_segment->ulength().value())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapElementSegmentOutOfBounds);
}
wasm::WireBytesRef segment_source =
instance->data_segments()->get(data_segment_index);
if (!base::IsInBounds<uint32_t>(data_start, data_length,
segment_source.length())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapDataSegmentOutOfBounds);
}
base::Vector<const uint8_t> data =
instance->native_module()->wire_bytes().SubVector(
segment_source.offset() + data_start,
segment_source.offset() + data_start + data_length);
PrototypesSetup_Sections decoder(
isolate, data, prototypes_segment, prototypes_start, prototypes_length,
functions_segment, functions_start, functions_length);
return decoder.SetupPrototypes(constructors);
}
#define RETURN_RESULT_OR_TRAP(call) \
do { \
DirectHandle<Object> result; \
if (!(call).ToHandle(&result)) { \
DCHECK(isolate->has_exception()); \
/* Mark any exception as uncatchable by Wasm. */ \
DirectHandle<JSObject> exception(Cast<JSObject>(isolate->exception()), \
isolate); \
DirectHandle<Name> uncatchable = \
isolate->factory()->wasm_uncatchable_symbol(); \
LookupIterator it(isolate, exception, uncatchable, LookupIterator::OWN); \
if (!JSReceiver::HasProperty(&it).FromJust()) { \
JSObject::AddProperty(isolate, exception, uncatchable, \
isolate->factory()->true_value(), NONE); \
} \
return ReadOnlyRoots(isolate).exception(); \
} \
DCHECK(!isolate->has_exception() || \
IsTerminationException(isolate->exception())); \
return *result; \
} while (false)
// "Special" because the type must be in a recgroup of its own.
// Used by "JS String Builtins".
RUNTIME_FUNCTION(Runtime_WasmCastToSpecialPrimitiveArray) {
HandleScope scope(isolate);
DCHECK_EQ(3, args.length());
int bits = args.smi_value_at(1);
int shared = args.smi_value_at(2);
DCHECK(bits == 8 || bits == 16);
if (args[0] == ReadOnlyRoots(isolate).null_value()) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapNullDereference);
}
MessageTemplate illegal_cast = MessageTemplate::kWasmTrapIllegalCast;
if (!IsWasmArray(args[0])) return ThrowWasmError(isolate, illegal_cast);
Tagged<WasmArray> obj = Cast<WasmArray>(args[0]);
wasm::CanonicalTypeIndex expected =
bits == 8
? (shared ? wasm::TypeCanonicalizer::kPredefinedArrayI8SharedIndex
: wasm::TypeCanonicalizer::kPredefinedArrayI8Index)
: (shared ? wasm::TypeCanonicalizer::kPredefinedArrayI16SharedIndex
: wasm::TypeCanonicalizer::kPredefinedArrayI16Index);
Tagged<Object> expected_map =
MakeStrong(isolate->heap()->wasm_canonical_rtts()->get(expected.index));
// If the expected_map has been cleared or never even created, then there's
// no chance of a match anyway.
if (obj->map() != expected_map) {
return ThrowWasmError(isolate, illegal_cast);
}
return obj;
}
// Returns the new string if the operation succeeds. Otherwise throws an
// exception and returns an empty result.
RUNTIME_FUNCTION(Runtime_WasmStringNewWtf8) {
DCHECK_EQ(5, args.length());
HandleScope scope(isolate);
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t memory = args.positive_smi_value_at(1);
UnicodeConfig config(args.positive_smi_value_at(2));
double offset_double = args.number_value_at(3);
uintptr_t offset = static_cast<uintptr_t>(offset_double);
uint32_t size = NumberToUint32(args[4]);
uint64_t mem_size = trusted_instance_data->memory_size(memory);
if (!base::IsInBounds<uint64_t>(offset, size, mem_size)) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapMemOutOfBounds);
}
const base::Vector<const uint8_t> bytes{
trusted_instance_data->memory_base(memory) + offset, size};
MaybeDirectHandle<v8::internal::String> result_string =
isolate->factory()->NewStringFromUtf8(bytes, config);
if (config.variant() == unibrow::Utf8Variant::kUtf8NoTrap) {
// If the input was invalid, then the decoder has failed silently, and
// the string.new_utf8_try instruction should return null.
if (result_string.is_null() && !isolate->has_exception()) {
return *isolate->factory()->wasm_null();
}
// Fall through in case of a valid result, and in case of a pending
// exception because the requested string was too large.
}
RETURN_RESULT_OR_TRAP(result_string);
}
RUNTIME_FUNCTION(Runtime_WasmStringNewWtf8Array) {
DCHECK_EQ(4, args.length());
HandleScope scope(isolate);
UnicodeConfig config(args.positive_smi_value_at(0));
DirectHandle<WasmArray> array(Cast<WasmArray>(args[1]), isolate);
uint32_t start = NumberToUint32(args[2]);
uint32_t end = NumberToUint32(args[3]);
TSAN_ACQUIRE(array->address());
MaybeDirectHandle<v8::internal::String> result_string =
isolate->factory()->NewStringFromUtf8(array, start, end, config);
if (config.variant() == unibrow::Utf8Variant::kUtf8NoTrap) {
// If the input was invalid, then the decoder has failed silently, and
// the string.new_utf8_array_try instruction should return null.
if (result_string.is_null() && !isolate->has_exception()) {
return *isolate->factory()->wasm_null();
}
// Fall through in case of a valid result, and in case of a pending
// exception because the requested string was too large.
}
RETURN_RESULT_OR_TRAP(result_string);
}
RUNTIME_FUNCTION(Runtime_WasmStringNewWtf16) {
DCHECK_EQ(5, args.length());
HandleScope scope(isolate);
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t memory = args.positive_smi_value_at(1);
double offset_double = args.number_value_at(2);
uintptr_t offset = static_cast<uintptr_t>(offset_double);
uint32_t size_in_codeunits = NumberToUint32(args[3]);
UnicodeConfig config(args.positive_smi_value_at(4));
uint64_t mem_size = trusted_instance_data->memory_size(memory);
if (size_in_codeunits > kMaxUInt32 / 2 ||
!base::IsInBounds<uint64_t>(offset, size_in_codeunits * 2, mem_size)) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapMemOutOfBounds);
}
if (offset & 1) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapUnalignedAccess);
}
const uint8_t* bytes = trusted_instance_data->memory_base(memory) + offset;
const base::uc16* codeunits = reinterpret_cast<const base::uc16*>(bytes);
RETURN_RESULT_OR_TRAP(isolate->factory()->NewStringFromTwoByteLittleEndian(
{codeunits, size_in_codeunits}, config));
}
RUNTIME_FUNCTION(Runtime_WasmStringNewWtf16Array) {
DCHECK_EQ(4, args.length());
HandleScope scope(isolate);
DirectHandle<WasmArray> array(Cast<WasmArray>(args[0]), isolate);
uint32_t start = NumberToUint32(args[1]);
uint32_t end = NumberToUint32(args[2]);
UnicodeConfig config(args.positive_smi_value_at(3));
TSAN_ACQUIRE(array->address());
RETURN_RESULT_OR_TRAP(
isolate->factory()->NewStringFromUtf16(array, start, end, config));
}
RUNTIME_FUNCTION(Runtime_WasmSubstring) {
DCHECK_EQ(3, args.length());
HandleScope scope(isolate);
DirectHandle<String> string(Cast<String>(args[0]), isolate);
uint32_t start = args.positive_smi_value_at(1);
uint32_t length = args.positive_smi_value_at(2);
string = String::Flatten(isolate, string);
return *isolate->factory()->NewCopiedSubstring(string, start, length);
}
RUNTIME_FUNCTION(Runtime_WasmSubstringShared) {
DCHECK_EQ(3, args.length());
HandleScope scope(isolate);
DirectHandle<String> string(Cast<String>(args[0]), isolate);
uint32_t start = args.positive_smi_value_at(1);
uint32_t length = args.positive_smi_value_at(2);
// TSAN_ACQUIRE has been called in the Torque builtin.
return *isolate->factory()->NewCopiedSubstringShared(string, start, length);
}
// Returns the new string if the operation succeeds. Otherwise traps.
RUNTIME_FUNCTION(Runtime_WasmStringConst) {
DCHECK_EQ(2, args.length());
HandleScope scope(isolate);
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
static_assert(
base::IsInRange(wasm::kV8MaxWasmStringLiterals, 0, Smi::kMaxValue));
uint32_t index = args.positive_smi_value_at(1);
DCHECK_LT(index, trusted_instance_data->module()->stringref_literals.size());
const wasm::WasmStringRefLiteral& literal =
trusted_instance_data->module()->stringref_literals[index];
const base::Vector<const uint8_t> module_bytes =
trusted_instance_data->native_module()->wire_bytes();
const base::Vector<const uint8_t> string_bytes = module_bytes.SubVector(
literal.source.offset(), literal.source.end_offset());
// TODO(12868): No need to re-validate WTF-8. Also, result should be cached.
return *isolate->factory()
->NewStringFromUtf8(string_bytes, unibrow::Utf8Variant::kWtf8)
.ToHandleChecked();
}
RUNTIME_FUNCTION(Runtime_WasmStringNewSegmentWtf8) {
DCHECK_EQ(5, args.length());
HandleScope scope(isolate);
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t segment_index = args.positive_smi_value_at(1);
uint32_t offset = args.positive_smi_value_at(2);
uint32_t length = args.positive_smi_value_at(3);
unibrow::Utf8Variant variant =
static_cast<unibrow::Utf8Variant>(args.positive_smi_value_at(4));
wasm::WireBytesRef segment_source =
trusted_instance_data->data_segments()->get(segment_index);
if (!base::IsInBounds<uint32_t>(offset, length, segment_source.length())) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapDataSegmentOutOfBounds);
}
base::Vector<const uint8_t> source =
trusted_instance_data->native_module()->wire_bytes().SubVector(
segment_source.offset() + offset,
segment_source.offset() + offset + length);
MaybeDirectHandle<String> result =
isolate->factory()->NewStringFromUtf8(source, variant);
if (variant == unibrow::Utf8Variant::kUtf8NoTrap) {
// If the input was invalid, then the decoder has failed silently, and
// the string.new_utf8_array_try instruction should return null.
// Only instructions from the stringref proposal can set variant
// kUtf8NoTrap, so WasmNull is appropriate here.
if (result.is_null() && !isolate->has_exception()) {
return *isolate->factory()->wasm_null();
}
// Fall through in case of a valid result, and in case of a pending
// exception because the requested string was too large.
}
RETURN_RESULT_OR_FAILURE(isolate, result);
}
namespace {
// TODO(12868): Consider unifying with api.cc:String::Utf8Length.
template <typename T>
uint32_t MeasureWtf8(base::Vector<const T> wtf16) {
int previous = unibrow::Utf16::kNoPreviousCharacter;
uint32_t length = 0;
DCHECK(wtf16.size() <= String::kMaxLength);
static_assert(String::kMaxLength <=
(kMaxInt / unibrow::Utf8::kMaxEncodedSize));
for (size_t i = 0; i < wtf16.size(); i++) {
int current = wtf16[i];
length += unibrow::Utf8::Length(current, previous);
previous = current;
}
return length;
}
uint32_t MeasureWtf8(Isolate* isolate, DirectHandle<String> string) {
string = String::Flatten(isolate, string);
DisallowGarbageCollection no_gc;
String::FlatContent content = string->GetFlatContent(no_gc);
DCHECK(content.IsFlat());
return content.IsOneByte() ? MeasureWtf8(content.ToOneByteVector())
: MeasureWtf8(content.ToUC16Vector());
}
size_t MaxEncodedSize(base::Vector<const uint8_t> wtf16) {
DCHECK(wtf16.size() < std::numeric_limits<size_t>::max() /
unibrow::Utf8::kMax8BitCodeUnitSize);
return wtf16.size() * unibrow::Utf8::kMax8BitCodeUnitSize;
}
size_t MaxEncodedSize(base::Vector<const base::uc16> wtf16) {
DCHECK(wtf16.size() < std::numeric_limits<size_t>::max() /
unibrow::Utf8::kMax16BitCodeUnitSize);
return wtf16.size() * unibrow::Utf8::kMax16BitCodeUnitSize;
}
bool HasUnpairedSurrogate(base::Vector<const uint8_t> wtf16) { return false; }
bool HasUnpairedSurrogate(base::Vector<const base::uc16> wtf16) {
return unibrow::Utf16::HasUnpairedSurrogate(wtf16.begin(), wtf16.size());
}
template <typename T>
int EncodeWtf8(base::Vector<char> bytes, size_t offset,
base::Vector<const T> wtf16, unibrow::Utf8Variant variant,
MessageTemplate* message, MessageTemplate out_of_bounds) {
// The first check is a quick estimate to decide whether the second check
// is worth the computation.
if (!base::IsInBounds<size_t>(offset, MaxEncodedSize(wtf16), bytes.size()) &&
!base::IsInBounds<size_t>(offset, MeasureWtf8(wtf16), bytes.size())) {
*message = out_of_bounds;
return -1;
}
bool replace_invalid = false;
switch (variant) {
case unibrow::Utf8Variant::kWtf8:
break;
case unibrow::Utf8Variant::kUtf8:
if (HasUnpairedSurrogate(wtf16)) {
*message = MessageTemplate::kWasmTrapStringIsolatedSurrogate;
return -1;
}
break;
case unibrow::Utf8Variant::kLossyUtf8:
replace_invalid = true;
break;
default:
UNREACHABLE();
}
bool write_null = false;
unibrow::Utf8::EncodingResult result =
unibrow::Utf8::Encode(wtf16, bytes.begin() + offset,
bytes.size() - offset, write_null, replace_invalid);
DCHECK_EQ(result.characters_processed, wtf16.size());
DCHECK_LE(result.bytes_written, kMaxInt);
return static_cast<int>(result.bytes_written);
}
template <typename GetWritableBytes>
Tagged<Object> EncodeWtf8(Isolate* isolate, unibrow::Utf8Variant variant,
DirectHandle<String> string,
GetWritableBytes get_writable_bytes, size_t offset,
MessageTemplate out_of_bounds_message) {
string = String::Flatten(isolate, string);
MessageTemplate message;
int written;
{
DisallowGarbageCollection no_gc;
String::FlatContent content = string->GetFlatContent(no_gc);
base::Vector<char> dst = get_writable_bytes(no_gc);
written = content.IsOneByte()
? EncodeWtf8(dst, offset, content.ToOneByteVector(), variant,
&message, out_of_bounds_message)
: EncodeWtf8(dst, offset, content.ToUC16Vector(), variant,
&message, out_of_bounds_message);
}
if (written < 0) {
DCHECK_NE(message, MessageTemplate::kNone);
return ThrowWasmError(isolate, message);
}
return *isolate->factory()->NewNumberFromInt(written);
}
} // namespace
// Used for storing the name of a string-constants imports module off the heap.
// Defined here to be able to make use of the helper functions above.
void ToUtf8Lossy(Isolate* isolate, DirectHandle<String> string,
std::string& out) {
uint32_t utf8_length = MeasureWtf8(isolate, string);
DisallowGarbageCollection no_gc;
out.resize(utf8_length);
String::FlatContent content = string->GetFlatContent(no_gc);
DCHECK(content.IsFlat());
static constexpr unibrow::Utf8Variant variant =
unibrow::Utf8Variant::kLossyUtf8;
MessageTemplate* error_cant_happen = nullptr;
MessageTemplate oob_cant_happen = MessageTemplate::kInvalid;
if (content.IsOneByte()) {
EncodeWtf8({out.data(), out.size()}, 0, content.ToOneByteVector(), variant,
error_cant_happen, oob_cant_happen);
} else {
EncodeWtf8({out.data(), out.size()}, 0, content.ToUC16Vector(), variant,
error_cant_happen, oob_cant_happen);
}
}
RUNTIME_FUNCTION(Runtime_WasmStringMeasureUtf8) {
DCHECK_EQ(1, args.length());
HandleScope scope(isolate);
DirectHandle<String> string(Cast<String>(args[0]), isolate);
TSAN_ACQUIRE(string->address());
string = String::Flatten(isolate, string);
uint32_t length;
{
DisallowGarbageCollection no_gc;
String::FlatContent content = string->GetFlatContent(no_gc);
DCHECK(content.IsFlat());
if (content.IsOneByte()) {
length = MeasureWtf8(content.ToOneByteVector());
} else {
base::Vector<const base::uc16> code_units = content.ToUC16Vector();
if (unibrow::Utf16::HasUnpairedSurrogate(code_units.begin(),
code_units.size())) {
return Smi::FromInt(-1);
} else {
length = MeasureWtf8(code_units);
}
}
}
return *isolate->factory()->NewNumberFromUint(length);
}
RUNTIME_FUNCTION(Runtime_WasmStringMeasureWtf8) {
DCHECK_EQ(1, args.length());
HandleScope scope(isolate);
DirectHandle<String> string(Cast<String>(args[0]), isolate);
TSAN_ACQUIRE(string->address());
uint32_t length = MeasureWtf8(isolate, string);
return *isolate->factory()->NewNumberFromUint(length);
}
RUNTIME_FUNCTION(Runtime_WasmStringEncodeWtf8) {
DCHECK_EQ(5, args.length());
HandleScope scope(isolate);
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
uint32_t memory = args.positive_smi_value_at(1);
uint32_t utf8_variant_value = args.positive_smi_value_at(2);
DirectHandle<String> string(Cast<String>(args[3]), isolate);
double offset_double = args.number_value_at(4);
uintptr_t offset = static_cast<uintptr_t>(offset_double);
DCHECK(utf8_variant_value <=
static_cast<uint32_t>(unibrow::Utf8Variant::kLastUtf8Variant));
auto utf8_variant = static_cast<unibrow::Utf8Variant>(utf8_variant_value);
auto get_writable_bytes =
[&](const DisallowGarbageCollection&) -> base::Vector<char> {
return {reinterpret_cast<char*>(trusted_instance_data->memory_base(memory)),
trusted_instance_data->memory_size(memory)};
};
return EncodeWtf8(isolate, utf8_variant, string, get_writable_bytes, offset,
MessageTemplate::kWasmTrapMemOutOfBounds);
}
namespace {
DISABLE_TSAN Tagged<Object> WasmStringEncodeWtf8ArrayImpl(
Isolate* isolate, uint32_t utf8_variant_value, DirectHandle<String> string,
DirectHandle<WasmArray> array, uint32_t start) {
DCHECK(utf8_variant_value <=
static_cast<uint32_t>(unibrow::Utf8Variant::kLastUtf8Variant));
auto utf8_variant = static_cast<unibrow::Utf8Variant>(utf8_variant_value);
auto get_writable_bytes =
[&](const DisallowGarbageCollection&) -> base::Vector<char> {
return {reinterpret_cast<char*>(array->ElementAddress(0)), array->length()};
};
return EncodeWtf8(isolate, utf8_variant, string, get_writable_bytes, start,
MessageTemplate::kWasmTrapArrayOutOfBounds);
}
} // namespace
RUNTIME_FUNCTION(Runtime_WasmStringEncodeWtf8Array) {
DCHECK_EQ(4, args.length());
HandleScope scope(isolate);
uint32_t utf8_variant_value = args.positive_smi_value_at(0);
DirectHandle<String> string(Cast<String>(args[1]), isolate);
DirectHandle<WasmArray> array(Cast<WasmArray>(args[2]), isolate);
uint32_t start = NumberToUint32(args[3]);
return WasmStringEncodeWtf8ArrayImpl(isolate, utf8_variant_value, string,
array, start);
}
RUNTIME_FUNCTION(Runtime_WasmStringToUtf8Array) {
DCHECK_EQ(2, args.length());
HandleScope scope(isolate);
DirectHandle<String> string(Cast<String>(args[0]), isolate);
int32_t shared = args.smi_value_at(1);
TSAN_ACQUIRE(string->address());
uint32_t length = MeasureWtf8(isolate, string);
constexpr int kElemSize = wasm::kWasmI8.value_kind_size();
if (length > static_cast<uint32_t>(WasmArray::MaxLength(kElemSize))) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapArrayTooLarge);
}
wasm::WasmValue initial_value(int8_t{0});
Tagged<WeakFixedArray> rtts = isolate->heap()->wasm_canonical_rtts();
// This function can only get called from Wasm code, so we can safely assume
// that the canonical RTT is still around.
DirectHandle<Map> map(
Cast<Map>(
rtts->get(
shared
? wasm::TypeCanonicalizer::kPredefinedArrayI8SharedIndex
.index
: wasm::TypeCanonicalizer::kPredefinedArrayI8Index.index)
.GetHeapObjectAssumeWeak()),
isolate);
AllocationType allocation =
shared ? AllocationType::kSharedOld : AllocationType::kYoung;
DirectHandle<WasmArray> array =
isolate->factory()->NewWasmArray(wasm::kWasmI8, length, initial_value,
map, allocation, SKIP_WRITE_BARRIER);
auto get_writable_bytes =
[&](const DisallowGarbageCollection&) -> base::Vector<char> {
return {reinterpret_cast<char*>(array->ElementAddress(0)), length};
};
Tagged<Object> write_result =
EncodeWtf8(isolate, unibrow::Utf8Variant::kLossyUtf8, string,
get_writable_bytes, 0, MessageTemplate::kNone);
DCHECK(IsNumber(write_result) && Object::NumberValue(write_result) == length);
USE(write_result);
return *array;
}
RUNTIME_FUNCTION(Runtime_WasmStringEncodeWtf16) {
DCHECK_EQ(6, args.length());
HandleScope scope(isolate);
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t memory = args.positive_smi_value_at(1);
Tagged<String> string = Cast<String>(args[2]);
double offset_double = args.number_value_at(3);
uintptr_t offset = static_cast<uintptr_t>(offset_double);
uint32_t start = args.positive_smi_value_at(4);
uint32_t length = args.positive_smi_value_at(5);
TSAN_ACQUIRE(string.address());
DCHECK(base::IsInBounds<uint32_t>(start, length, string->length()));
size_t mem_size = trusted_instance_data->memory_size(memory);
static_assert(String::kMaxLength <=
(std::numeric_limits<size_t>::max() / sizeof(base::uc16)));
if (!base::IsInBounds<size_t>(offset, length * sizeof(base::uc16),
mem_size)) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapMemOutOfBounds);
}
if (offset & 1) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapUnalignedAccess);
}
#if defined(V8_TARGET_LITTLE_ENDIAN)
uint16_t* dst = reinterpret_cast<uint16_t*>(
trusted_instance_data->memory_base(memory) + offset);
String::WriteToFlat(string, dst, start, length);
return Smi::zero(); // Unused.
#elif defined(V8_TARGET_BIG_ENDIAN)
// TODO(12868): The host is big-endian but we need to write the string
// contents as little-endian.
USE(string);
USE(start);
UNIMPLEMENTED();
#else
#error Unknown endianness
#endif
}
RUNTIME_FUNCTION(Runtime_WasmStringAsWtf8) {
DCHECK_EQ(1, args.length());
HandleScope scope(isolate);
DirectHandle<String> string(Cast<String>(args[0]), isolate);
TSAN_ACQUIRE(string->address());
uint32_t wtf8_length = MeasureWtf8(isolate, string);
DirectHandle<ByteArray> array = isolate->factory()->NewByteArray(wtf8_length);
auto utf8_variant = unibrow::Utf8Variant::kWtf8;
auto get_writable_bytes =
[&](const DisallowGarbageCollection&) -> base::Vector<char> {
return {reinterpret_cast<char*>(array->begin()), wtf8_length};
};
EncodeWtf8(isolate, utf8_variant, string, get_writable_bytes, 0,
MessageTemplate::kWasmTrapArrayOutOfBounds);
return *array;
}
RUNTIME_FUNCTION(Runtime_WasmStringViewWtf8Encode) {
DCHECK_EQ(7, args.length());
HandleScope scope(isolate);
Tagged<WasmTrustedInstanceData> trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(args[0]);
uint32_t utf8_variant_value = args.positive_smi_value_at(1);
DirectHandle<ByteArray> array(Cast<ByteArray>(args[2]), isolate);
double addr_double = args.number_value_at(3);
uintptr_t addr = static_cast<uintptr_t>(addr_double);
uint32_t start = NumberToUint32(args[4]);
uint32_t end = NumberToUint32(args[5]);
uint32_t memory = args.positive_smi_value_at(6);
DCHECK(utf8_variant_value <=
static_cast<uint32_t>(unibrow::Utf8Variant::kLastUtf8Variant));
DCHECK_LE(start, end);
DCHECK(
base::IsInBounds<size_t>(start, end - start, array->ulength().value()));
auto utf8_variant = static_cast<unibrow::Utf8Variant>(utf8_variant_value);
size_t length = end - start;
if (!base::IsInBounds<size_t>(addr, length,
trusted_instance_data->memory_size(memory))) {
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapMemOutOfBounds);
}
uint8_t* memory_start = trusted_instance_data->memory_base(memory);
const uint8_t* src = reinterpret_cast<const uint8_t*>(array->begin() + start);
uint8_t* dst = memory_start + addr;
std::vector<size_t> surrogates;
if (utf8_variant != unibrow::Utf8Variant::kWtf8) {
unibrow::Wtf8::ScanForSurrogates({src, length}, &surrogates);
if (utf8_variant == unibrow::Utf8Variant::kUtf8 && !surrogates.empty()) {
return ThrowWasmError(isolate,
MessageTemplate::kWasmTrapStringIsolatedSurrogate);
}
}
MemCopy(dst, src, length);
for (size_t surrogate : surrogates) {
DCHECK_LT(surrogate, length);
DCHECK_EQ(utf8_variant, unibrow::Utf8Variant::kLossyUtf8);
unibrow::Utf8::Encode(reinterpret_cast<char*>(dst + surrogate),
unibrow::Utf8::kBadChar, 0, false);
}
// Unused.
return Tagged<Smi>(0);
}
RUNTIME_FUNCTION(Runtime_WasmStringViewWtf8Slice) {
DCHECK_EQ(3, args.length());
HandleScope scope(isolate);
DirectHandle<ByteArray> array(Cast<ByteArray>(args[0]), isolate);
uint32_t start = NumberToUint32(args[1]);
uint32_t end = NumberToUint32(args[2]);
DCHECK_LT(start, end);
DCHECK(
base::IsInBounds<size_t>(start, end - start, array->ulength().value()));
// This can't throw because the result can't be too long if the input wasn't,
// and encoding failures are ruled out too because {start}/{end} are aligned.
return *isolate->factory()
->NewStringFromUtf8(array, start, end,
unibrow::Utf8Variant::kWtf8)
.ToHandleChecked();
}
#ifdef V8_ENABLE_DRUMBRAKE
RUNTIME_FUNCTION(Runtime_WasmTraceBeginExecution) {
SealHandleScope shs(isolate);
DisallowGarbageCollection no_gc;
DCHECK(v8_flags.slow_histograms && !v8_flags.wasm_jitless &&
v8_flags.wasm_enable_exec_time_histograms);
DCHECK_EQ(0, args.length());
wasm::WasmExecutionTimer* timer = isolate->wasm_execution_timer();
timer->Start();
return ReadOnlyRoots(isolate).undefined_value();
}
RUNTIME_FUNCTION(Runtime_WasmTraceEndExecution) {
SealHandleScope shs(isolate);
DisallowGarbageCollection no_gc;
DCHECK(v8_flags.slow_histograms && !v8_flags.wasm_jitless &&
v8_flags.wasm_enable_exec_time_histograms);
DCHECK_EQ(0, args.length());
wasm::WasmExecutionTimer* timer = isolate->wasm_execution_timer();
timer->Stop();
return ReadOnlyRoots(isolate).undefined_value();
}
#endif // V8_ENABLE_DRUMBRAKE
RUNTIME_FUNCTION(Runtime_WasmStringFromCodePoint) {
DCHECK_EQ(2, args.length());
HandleScope scope(isolate);
uint32_t code_point = NumberToUint32(args[0]);
int shared = args.smi_value_at(1);
// `LookupSingleCharacterStringFromCode` might return an internalized string
// if `code_point` is not one-byte, which is not shared.
if ((!shared && code_point <= unibrow::Utf16::kMaxNonSurrogateCharCode) ||
(shared && code_point <= String::kMaxOneByteCharCode)) {
return *isolate->factory()->LookupSingleCharacterStringFromCode(code_point);
}
if (code_point > 0x10FFFF) {
// Allocate a new number to preserve the to-uint conversion (e.g. if
// args[0] == -1, we want the error message to report 4294967295).
return ThrowWasmError(isolate, MessageTemplate::kInvalidCodePoint,
{isolate->factory()->NewNumberFromUint(code_point)});
}
if (shared && code_point <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
DirectHandle<SeqTwoByteString> result =
isolate->factory()->NewRawSharedTwoByteString(1).ToHandleChecked();
DisallowGarbageCollection no_gc;
CopyChars(result->GetChars(no_gc), &code_point, 1);
return *result;
}
base::uc16 char_buffer[] = {
unibrow::Utf16::LeadSurrogate(code_point),
unibrow::Utf16::TrailSurrogate(code_point),
};
DirectHandle<SeqTwoByteString> result =
(shared ? isolate->factory()->NewRawSharedTwoByteString(
arraysize(char_buffer))
: isolate->factory()->NewRawTwoByteString(arraysize(char_buffer)))
.ToHandleChecked();
DisallowGarbageCollection no_gc;
CopyChars(result->GetChars(no_gc), char_buffer, arraysize(char_buffer));
return *result;
}
RUNTIME_FUNCTION(Runtime_WasmStringHash) {
SealHandleScope seal_handle_scope(isolate);
DisallowGarbageCollection no_gc;
DCHECK_EQ(1, args.length());
Tagged<String> string(Cast<String>(args[0]));
uint32_t hash = string->EnsureHash();
return Smi::FromInt(static_cast<int>(hash));
}
RUNTIME_FUNCTION(Runtime_WasmStringAdd_NoMapCheck_Shared) {
DCHECK_EQ(2, args.length());
HandleScope scope(isolate);
DirectHandle<String> left(Cast<String>(args[0]), isolate);
DirectHandle<String> right(Cast<String>(args[1]), isolate);
TSAN_ACQUIRE(left->address());
TSAN_ACQUIRE(right->address());
DirectHandle<String> result;
if (isolate->factory()->WasmStringAddShared(left, right).ToHandle(&result)) {
return *result;
} else {
DCHECK(isolate->has_exception());
return ReadOnlyRoots(isolate).exception();
}
}
// For cont.new: this initializes the continuation with a new stack and with the
// given function reference, such that calling "resume" on it will call the
// function on the new stack.
RUNTIME_FUNCTION(Runtime_WasmAllocateContinuation) {
DCHECK_EQ(3, args.length());
HandleScope scope(isolate);
DirectHandle<WasmTrustedInstanceData> trusted_instance_data(
TrustedCast<WasmTrustedInstanceData>(args[0]), isolate);
DirectHandle<WasmFuncRef> func_ref(Cast<WasmFuncRef>(args[1]), isolate);
wasm::CanonicalTypeIndex sig_id{static_cast<uint32_t>(args.smi_value_at(2))};
std::unique_ptr<wasm::StackMemory> stack =
isolate->stack_pool().GetOrAllocate();
DirectHandle<WasmStackObject> stack_obj =
isolate->factory()->NewWasmStackObject(stack.get());
const wasm::CanonicalSig* sig =
wasm::GetTypeCanonicalizer()->LookupFunctionSignature(sig_id);
auto [arg_buffer_size, alignment] =
GetBufferSizeAndAlignmentFor(sig->parameters());
#if V8_TARGET_ARCH_ARM64
// For stack alignment.
alignment = RoundUp(alignment, 2 * kSystemPointerSize);
#endif
stack->jmpbuf()->fp = kNullAddress;
stack->jmpbuf()->sp = RoundDown(stack->base() - arg_buffer_size, alignment);
Address arg_buffer = stack->jmpbuf()->sp;
stack->set_arg_buffer(arg_buffer);
stack->jmpbuf()->state = wasm::JumpBuffer::Suspended;
stack->jmpbuf()->stack_limit = stack->jslimit();
stack->jmpbuf()->is_on_central_stack = false;
stack->jmpbuf()->parent = nullptr;
stack->set_index(isolate->wasm_stacks().size());
// TODO(thibaudm): Store the WasmCodePointer instead.
wasm::StackEntryWrapperCacheKey key{sig};
std::shared_ptr<wasm::WasmWrapperHandle> wrapper =
wasm::GetWasmStackEntryWrapperCache()->GetCompiled(isolate, key);
stack->jmpbuf()->pc = wrapper->code()->instruction_start();
trusted_instance_data->native_module()->RegisterStackEntryWrapper(
std::move(wrapper));
stack->set_param_types(sig->parameters());
stack->set_signature_id(sig->index());
DirectHandle<WasmContinuationObject> cont =
isolate->factory()->NewWasmContinuationObject(stack_obj);
// Set the references after the heap allocation, so that they are not
// immediately stale from a potential GC.
stack->set_func_ref(*func_ref);
stack->set_current_continuation(*cont);
stack->set_stack_obj(*stack_obj);
isolate->wasm_stacks().emplace_back(std::move(stack));
return *cont;
}
RUNTIME_FUNCTION(Runtime_WasmTypeAssertionFailed) {
DCHECK_EQ(0, args.length());
// The "FuzzerSecurityIssueHigh" is needed to label crashes of this as
// security issues in ClusterFuzz.
FATAL("[FuzzerSecurityIssueHigh] Wasm type assertion violation");
}
#ifdef V8_IS_TSAN
// Since TSAN does not know about release fences, we must manually define the
// synchronization between object initialization and reads from that object. We
// add a TSAN_RELEASE after the object-initialization release fence, and a
// TSAN_ACQUIRE at the beginning of each read-only builtin (future work: also,
// before every read of a shared object in generated code). Explainer:
// https://docs.google.com/document/d/17RLOdAFJ2HFA4hE83wSsTYdRwHV4ZBiX_jtOUp0qatw/edit?usp=sharing
RUNTIME_FUNCTION(Runtime_TsanAcquireForInitializationFence) {
DCHECK_EQ(1, args.length());
TSAN_ACQUIRE(Cast<HeapObject>(args[0]).address());
return ReadOnlyRoots(isolate).undefined_value();
}
#endif // V8_IS_TSAN
#undef RuntimeArguments
} // namespace v8::internal