// Copyright 2014 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <assert.h> // For assert
#include <limits.h> // For LONG_MIN, LONG_MAX.
#if V8_TARGET_ARCH_PPC64
#include <optional>
#include "src/base/bits.h"
#include "src/base/division-by-constant.h"
#include "src/builtins/builtins-inl.h"
#include "src/codegen/callable.h"
#include "src/codegen/code-factory.h"
#include "src/codegen/external-reference-table.h"
#include "src/codegen/interface-descriptors-inl.h"
#include "src/codegen/macro-assembler.h"
#include "src/codegen/register-configuration.h"
#include "src/codegen/register.h"
#include "src/debug/debug.h"
#include "src/deoptimizer/deoptimizer.h"
#include "src/execution/frames-inl.h"
#include "src/heap/mutable-page.h"
#include "src/init/bootstrapper.h"
#include "src/logging/counters.h"
#include "src/runtime/runtime.h"
#include "src/snapshot/snapshot.h"
// Satisfy cpplint check, but don't include platform-specific header. It is
// included recursively via macro-assembler.h.
#if 0
#include "src/codegen/ppc/macro-assembler-ppc.h"
#endif
#define __ ACCESS_MASM(masm)
namespace v8 {
namespace internal {
namespace {
// Simd and Floating Pointer registers are not shared. For WebAssembly we save
// both registers, If we are not running Wasm, we can get away with only saving
// FP registers.
#if V8_ENABLE_WEBASSEMBLY
constexpr int kStackSavedSavedFPSizeInBytes =
(kNumCallerSavedDoubles * kSimd128Size) +
(kNumCallerSavedDoubles * kDoubleSize);
#else
constexpr int kStackSavedSavedFPSizeInBytes =
kNumCallerSavedDoubles * kDoubleSize;
#endif // V8_ENABLE_WEBASSEMBLY
} // namespace
int MacroAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
Register exclusion1,
Register exclusion2,
Register exclusion3) const {
int bytes = 0;
RegList exclusions = {exclusion1, exclusion2, exclusion3};
RegList list = kJSCallerSaved - exclusions;
bytes += list.Count() * kSystemPointerSize;
if (fp_mode == SaveFPRegsMode::kSave) {
bytes += kStackSavedSavedFPSizeInBytes;
}
return bytes;
}
int MacroAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
Register exclusion2, Register exclusion3) {
int bytes = 0;
RegList exclusions = {exclusion1, exclusion2, exclusion3};
RegList list = kJSCallerSaved - exclusions;
MultiPush(list);
bytes += list.Count() * kSystemPointerSize;
if (fp_mode == SaveFPRegsMode::kSave) {
MultiPushF64AndV128(kCallerSavedDoubles, kCallerSavedSimd128s);
bytes += kStackSavedSavedFPSizeInBytes;
}
return bytes;
}
int MacroAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
Register exclusion2, Register exclusion3) {
int bytes = 0;
if (fp_mode == SaveFPRegsMode::kSave) {
MultiPopF64AndV128(kCallerSavedDoubles, kCallerSavedSimd128s);
bytes += kStackSavedSavedFPSizeInBytes;
}
RegList exclusions = {exclusion1, exclusion2, exclusion3};
RegList list = kJSCallerSaved - exclusions;
MultiPop(list);
bytes += list.Count() * kSystemPointerSize;
return bytes;
}
void MacroAssembler::GetLabelAddress(Register dest, Label* target) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this);
// This should be just a
// add(dest, pc, branch_offset(target));
// but current implementation of Assembler::bind_to()/target_at_put() add
// (InstructionStream::kHeaderSize - kHeapObjectTag) to a position of a label
// in a "linked" state and thus making it usable only for mov_label_offset().
// TODO(ishell): fix branch_offset() and re-implement
// regexp::RegExpMacroAssemblerARM::PushBacktrack() without
// mov_label_offset().
mov_label_offset(dest, target);
// mov_label_offset computes offset of the |target| relative to the "current
// InstructionStream object pointer" which is essentially pc_offset() of the
// label added with (InstructionStream::kHeaderSize - kHeapObjectTag).
// Compute "current InstructionStream object pointer" and add it to the
// offset in |lr| register.
int current_instr_code_object_relative_offset =
pc_offset() + kPcLoadDelta +
(InstructionStream::kHeaderSize - kHeapObjectTag);
LoadPC(scratch);
AddS64(dest, scratch, dest);
// TODO(miladfarca): Use subi once scratch register usage is refactored
// and r0 can't be used as scratch or dest.
mov(scratch, Operand(current_instr_code_object_relative_offset));
SubS64(dest, dest, scratch);
}
void MacroAssembler::Jump(Register target) {
mtctr(target);
bctr();
}
void MacroAssembler::LoadFromConstantsTable(Register destination,
int constant_index) {
DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
DCHECK_NE(destination, r0);
LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
LoadTaggedField(destination,
FieldMemOperand(destination, FixedArray::OffsetOfElementAt(
constant_index)));
}
void MacroAssembler::LoadRootRelative(Register destination, int32_t offset) {
LoadU64(destination, MemOperand(kRootRegister, offset));
}
void MacroAssembler::StoreRootRelative(int32_t offset, Register value) {
StoreU64(value, MemOperand(kRootRegister, offset));
}
void MacroAssembler::LoadRootRegisterOffset(Register destination,
intptr_t offset) {
if (offset == 0) {
mr(destination, kRootRegister);
} else {
AddS64(destination, kRootRegister, Operand(offset));
}
}
MemOperand MacroAssembler::ExternalReferenceAsOperand(
ExternalReference reference, Register scratch) {
if (root_array_available()) {
if (reference.IsIsolateFieldId()) {
return MemOperand(kRootRegister, reference.offset_from_root_register());
}
if (options().enable_root_relative_access) {
intptr_t offset =
RootRegisterOffsetForExternalReference(isolate(), reference);
if (is_int32(offset)) {
return MemOperand(kRootRegister, static_cast<int32_t>(offset));
}
}
if (options().isolate_independent_code) {
if (IsAddressableThroughRootRegister(isolate(), reference)) {
// Some external references can be efficiently loaded as an offset from
// kRootRegister.
intptr_t offset =
RootRegisterOffsetForExternalReference(isolate(), reference);
CHECK(is_int32(offset));
return MemOperand(kRootRegister, static_cast<int32_t>(offset));
} else {
// Otherwise, do a memory load from the external reference table.
LoadU64(scratch,
MemOperand(kRootRegister,
RootRegisterOffsetForExternalReferenceTableEntry(
isolate(), reference)));
return MemOperand(scratch, 0);
}
}
}
Move(scratch, reference);
return MemOperand(scratch, 0);
}
void MacroAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
Condition cond, CRegister cr) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
if (cond != al) b(NegateCondition(cond), &skip, cr);
mov(scratch, Operand(target, rmode));
mtctr(scratch);
bctr();
bind(&skip);
}
void MacroAssembler::Jump(Address target, RelocInfo::Mode rmode, Condition cond,
CRegister cr) {
DCHECK(!RelocInfo::IsCodeTarget(rmode));
Jump(static_cast<intptr_t>(target), rmode, cond, cr);
}
void MacroAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
Condition cond, CRegister cr) {
DCHECK(RelocInfo::IsCodeTarget(rmode));
DCHECK_IMPLIES(options().isolate_independent_code,
Builtins::IsIsolateIndependentBuiltin(*code));
Builtin builtin = Builtin::kNoBuiltinId;
if (isolate()->builtins()->IsBuiltinHandle(code, &builtin)) {
TailCallBuiltin(builtin, cond, cr);
return;
}
int32_t target_index = AddCodeTarget(code);
Jump(static_cast<intptr_t>(target_index), rmode, cond, cr);
}
void MacroAssembler::Jump(const ExternalReference& reference) {
// ELFv2 ABI requires r12 to hold the callee address at the global entry
// point so the function can compute its TOC pointer. Use ip (r12)
// directly instead of acquiring a scratch register.
Move(ip, reference);
if (ABI_USES_FUNCTION_DESCRIPTORS) {
// AIX uses a function descriptor. When calling C code be
// aware of this descriptor and pick up values from it.
LoadU64(ToRegister(ABI_TOC_REGISTER), MemOperand(ip, kSystemPointerSize));
LoadU64(ip, MemOperand(ip, 0));
}
Jump(ip);
}
void MacroAssembler::Call(Register target) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// branch via link register and set LK bit for return point
mtctr(target);
bctrl();
RecordPcForSafepoint();
}
void MacroAssembler::CallJSEntry(Register target) {
CHECK(target == r5);
Call(target);
}
int MacroAssembler::CallSizeNotPredictableCodeSize(Address target,
RelocInfo::Mode rmode,
Condition cond) {
return (2 + kMovInstructionsNoConstantPool) * kInstrSize;
}
void MacroAssembler::Call(Address target, RelocInfo::Mode rmode,
Condition cond) {
BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(cond == al);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// This can likely be optimized to make use of bc() with 24bit relative
//
// RecordRelocInfo(x.rmode_, x.immediate);
// bc( BA, .... offset, LKset);
//
mov(scratch, Operand(target, rmode));
mtctr(scratch);
bctrl();
RecordPcForSafepoint();
}
void MacroAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
Condition cond) {
BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(RelocInfo::IsCodeTarget(rmode));
DCHECK_IMPLIES(options().isolate_independent_code,
Builtins::IsIsolateIndependentBuiltin(*code));
Builtin builtin = Builtin::kNoBuiltinId;
if (isolate()->builtins()->IsBuiltinHandle(code, &builtin)) {
CallBuiltin(builtin, cond);
return;
}
int32_t target_index = AddCodeTarget(code);
Call(static_cast<Address>(target_index), rmode, cond);
}
void MacroAssembler::CallBuiltin(Builtin builtin, Condition cond) {
ASM_CODE_COMMENT_STRING(this, CommentForOffHeapTrampoline("call", builtin));
switch (options().builtin_call_jump_mode) {
case BuiltinCallJumpMode::kAbsolute: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
mov(scratch, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));
if (cond != al) b(NegateCondition(cond), &skip);
Call(scratch);
bind(&skip);
break;
}
case BuiltinCallJumpMode::kPCRelative:
UNREACHABLE();
case BuiltinCallJumpMode::kIndirect: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
LoadU64(scratch, EntryFromBuiltinAsOperand(builtin));
if (cond != al) b(NegateCondition(cond), &skip);
Call(scratch);
bind(&skip);
break;
}
case BuiltinCallJumpMode::kForMksnapshot: {
if (options().use_pc_relative_calls_and_jumps_for_mksnapshot) {
Handle<Code> code = isolate()->builtins()->code_handle(builtin);
int32_t code_target_index = AddCodeTarget(code);
Call(static_cast<Address>(code_target_index), RelocInfo::CODE_TARGET,
cond);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
LoadU64(scratch, EntryFromBuiltinAsOperand(builtin));
if (cond != al) b(NegateCondition(cond), &skip);
Call(scratch);
bind(&skip);
}
break;
}
}
}
void MacroAssembler::TailCallBuiltin(Builtin builtin, Condition cond,
CRegister cr) {
ASM_CODE_COMMENT_STRING(this,
CommentForOffHeapTrampoline("tail call", builtin));
switch (options().builtin_call_jump_mode) {
case BuiltinCallJumpMode::kAbsolute: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
mov(scratch, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));
if (cond != al) b(NegateCondition(cond), &skip, cr);
Jump(scratch);
bind(&skip);
break;
}
case BuiltinCallJumpMode::kPCRelative:
UNREACHABLE();
case BuiltinCallJumpMode::kIndirect: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
LoadU64(scratch, EntryFromBuiltinAsOperand(builtin));
if (cond != al) b(NegateCondition(cond), &skip, cr);
Jump(scratch);
bind(&skip);
break;
}
case BuiltinCallJumpMode::kForMksnapshot: {
if (options().use_pc_relative_calls_and_jumps_for_mksnapshot) {
Handle<Code> code = isolate()->builtins()->code_handle(builtin);
int32_t code_target_index = AddCodeTarget(code);
Jump(static_cast<intptr_t>(code_target_index), RelocInfo::CODE_TARGET,
cond, cr);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label skip;
LoadU64(scratch, EntryFromBuiltinAsOperand(builtin));
if (cond != al) b(NegateCondition(cond), &skip, cr);
Jump(scratch);
bind(&skip);
}
break;
}
}
}
void MacroAssembler::Drop(int count) {
if (count > 0) {
AddS64(sp, sp, Operand(count * kSystemPointerSize));
}
}
void MacroAssembler::Drop(Register count) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ShiftLeftU64(scratch, count, Operand(kSystemPointerSizeLog2));
add(sp, sp, scratch);
}
// Enforce alignment of sp.
void MacroAssembler::EnforceStackAlignment() {
int frame_alignment = ActivationFrameAlignment();
DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
uint64_t frame_alignment_mask = ~(static_cast<uint64_t>(frame_alignment) - 1);
AndU64(sp, sp, Operand(frame_alignment_mask));
}
void MacroAssembler::TestCodeIsMarkedForDeoptimization(Register code) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadU32(scratch, FieldMemOperand(code, Code::kFlagsOffset));
TestBit(scratch, Code::kMarkedForDeoptimizationBit);
}
Operand MacroAssembler::ClearedValue() const {
return Operand(static_cast<int32_t>(i::kClearedWeakValue.ptr()));
}
void MacroAssembler::Call(Label* target) {
// Keep a trampoline pool from being emitted between the branch and the
// safepoint record, so pc_offset_for_safepoint_ stays matched to the return
// address. The Register and Address overloads rely on the same scope.
BlockTrampolinePoolScope block_trampoline_pool(this);
b(target, SetLK);
RecordPcForSafepoint();
}
void MacroAssembler::Push(Handle<HeapObject> handle) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(handle));
push(scratch);
}
void MacroAssembler::Push(Tagged<Smi> smi) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(smi));
push(scratch);
}
void MacroAssembler::Push(Tagged<TaggedIndex> index) {
// TaggedIndex is the same as Smi for 32 bit archs.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(static_cast<uint32_t>(index.value())));
push(scratch);
}
void MacroAssembler::PushArray(Register array, Register size, Register scratch,
Register scratch2, PushArrayOrder order) {
Label loop, done;
if (order == kNormal) {
cmpi(size, Operand::Zero());
beq(&done);
ShiftLeftU64(scratch, size, Operand(kSystemPointerSizeLog2));
add(scratch, array, scratch);
mtctr(size);
bind(&loop);
LoadU64WithUpdate(scratch2, MemOperand(scratch, -kSystemPointerSize));
StoreU64WithUpdate(scratch2, MemOperand(sp, -kSystemPointerSize));
bdnz(&loop);
bind(&done);
} else {
cmpi(size, Operand::Zero());
beq(&done);
mtctr(size);
subi(scratch, array, Operand(kSystemPointerSize));
bind(&loop);
LoadU64WithUpdate(scratch2, MemOperand(scratch, kSystemPointerSize));
StoreU64WithUpdate(scratch2, MemOperand(sp, -kSystemPointerSize));
bdnz(&loop);
bind(&done);
}
}
void MacroAssembler::Move(Register dst, Handle<HeapObject> value,
RelocInfo::Mode rmode) {
// TODO(jgruber,v8:8887): Also consider a root-relative load when generating
// non-isolate-independent code. In many cases it might be cheaper than
// embedding the relocatable value.
if (root_array_available_ && options().isolate_independent_code) {
IndirectLoadConstant(dst, value);
return;
} else if (RelocInfo::IsCompressedEmbeddedObject(rmode)) {
EmbeddedObjectIndex index = AddEmbeddedObject(value);
DCHECK(is_uint32(index));
mov(dst, Operand(static_cast<int>(index), rmode));
} else {
DCHECK(RelocInfo::IsFullEmbeddedObject(rmode));
mov(dst, Operand(value.address(), rmode));
}
}
void MacroAssembler::Move(Register dst, ExternalReference reference) {
if (root_array_available()) {
if (reference.IsIsolateFieldId()) {
AddS64(dst, kRootRegister,
Operand(reference.offset_from_root_register()));
return;
}
if (options().isolate_independent_code) {
IndirectLoadExternalReference(dst, reference);
return;
}
}
// External references should not get created with IDs if
// `!root_array_available()`.
CHECK(!reference.IsIsolateFieldId());
mov(dst, Operand(reference));
}
void MacroAssembler::LoadIsolateField(Register dst, IsolateFieldId id) {
Move(dst, ExternalReference::Create(id));
}
void MacroAssembler::Move(Register dst, Register src, Condition cond) {
DCHECK(cond == al);
if (dst != src) {
mr(dst, src);
}
}
void MacroAssembler::Move(DoubleRegister dst, DoubleRegister src) {
if (dst != src) {
fmr(dst, src);
}
}
void MacroAssembler::MultiPush(RegList regs, Register location) {
int16_t num_to_push = regs.Count();
int16_t stack_offset = num_to_push * kSystemPointerSize;
subi(location, location, Operand(stack_offset));
for (int16_t i = Register::kNumRegisters - 1; i >= 0; i--) {
if ((regs.bits() & (1 << i)) != 0) {
stack_offset -= kSystemPointerSize;
StoreU64(ToRegister(i), MemOperand(location, stack_offset));
}
}
}
void MacroAssembler::MultiPop(RegList regs, Register location) {
int16_t stack_offset = 0;
for (int16_t i = 0; i < Register::kNumRegisters; i++) {
if ((regs.bits() & (1 << i)) != 0) {
LoadU64(ToRegister(i), MemOperand(location, stack_offset));
stack_offset += kSystemPointerSize;
}
}
addi(location, location, Operand(stack_offset));
}
void MacroAssembler::MultiPushDoubles(DoubleRegList dregs, Register location) {
int16_t num_to_push = dregs.Count();
int16_t stack_offset = num_to_push * kDoubleSize;
subi(location, location, Operand(stack_offset));
for (int16_t i = DoubleRegister::kNumRegisters - 1; i >= 0; i--) {
if ((dregs.bits() & (1 << i)) != 0) {
DoubleRegister dreg = DoubleRegister::from_code(i);
stack_offset -= kDoubleSize;
stfd(dreg, MemOperand(location, stack_offset));
}
}
}
void MacroAssembler::MultiPushV128(Simd128RegList simd_regs,
Register location) {
int16_t num_to_push = simd_regs.Count();
int16_t stack_offset = num_to_push * kSimd128Size;
subi(location, location, Operand(stack_offset));
for (int16_t i = Simd128Register::kNumRegisters - 1; i >= 0; i--) {
if ((simd_regs.bits() & (1 << i)) != 0) {
Simd128Register simd_reg = Simd128Register::from_code(i);
stack_offset -= kSimd128Size;
StoreSimd128(simd_reg, MemOperand(location, stack_offset));
}
}
}
void MacroAssembler::MultiPopDoubles(DoubleRegList dregs, Register location) {
int16_t stack_offset = 0;
for (int16_t i = 0; i < DoubleRegister::kNumRegisters; i++) {
if ((dregs.bits() & (1 << i)) != 0) {
DoubleRegister dreg = DoubleRegister::from_code(i);
lfd(dreg, MemOperand(location, stack_offset));
stack_offset += kDoubleSize;
}
}
addi(location, location, Operand(stack_offset));
}
void MacroAssembler::MultiPopV128(Simd128RegList simd_regs, Register location) {
int16_t stack_offset = 0;
for (int16_t i = 0; i < Simd128Register::kNumRegisters; i++) {
if ((simd_regs.bits() & (1 << i)) != 0) {
Simd128Register simd_reg = Simd128Register::from_code(i);
LoadSimd128(simd_reg, MemOperand(location, stack_offset));
stack_offset += kSimd128Size;
}
}
addi(location, location, Operand(stack_offset));
}
void MacroAssembler::MultiPushF64AndV128(DoubleRegList dregs,
Simd128RegList simd_regs,
Register location) {
MultiPushDoubles(dregs, location);
#if V8_ENABLE_WEBASSEMBLY
MultiPushV128(simd_regs, location);
#endif
}
void MacroAssembler::MultiPopF64AndV128(DoubleRegList dregs,
Simd128RegList simd_regs,
Register location) {
#if V8_ENABLE_WEBASSEMBLY
MultiPopV128(simd_regs, location);
#endif
MultiPopDoubles(dregs, location);
}
void MacroAssembler::PushAll(RegList registers) {
if (registers.is_empty()) return;
ASM_CODE_COMMENT(this);
// TODO(victorgomes): {stm/ldm} pushes/pops registers in the opposite order
// as expected by Maglev frame. Consider massaging Maglev to accept this
// order instead.
// Can not use MultiPush(registers, sp) due to orders
for (Register reg : registers) {
Push(reg);
}
}
void MacroAssembler::PopAll(RegList registers) {
if (registers.is_empty()) return;
ASM_CODE_COMMENT(this);
// Can not use MultiPop(registers, sp);
for (Register reg : base::Reversed(registers)) {
Pop(reg);
}
}
void MacroAssembler::PushAll(DoubleRegList registers, int stack_slot_size) {
if (registers.is_empty()) return;
ASM_CODE_COMMENT(this);
MultiPushDoubles(registers, sp);
}
void MacroAssembler::PopAll(DoubleRegList registers, int stack_slot_size) {
if (registers.is_empty()) return;
ASM_CODE_COMMENT(this);
MultiPopDoubles(registers, sp);
}
void MacroAssembler::LoadTaggedRoot(Register destination, RootIndex index) {
ASM_CODE_COMMENT(this);
if (CanBeImmediate(index)) {
mov(destination, Operand(ReadOnlyRootPtr(index), RelocInfo::Mode::NO_INFO));
return;
}
LoadRoot(destination, index);
}
void MacroAssembler::StoreTaggedRoot(const MemOperand& destination,
RootIndex index) {
ASM_CODE_COMMENT(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadTaggedRoot(scratch, index);
StoreTaggedField(scratch, destination);
}
void MacroAssembler::LoadRoot(Register destination, RootIndex index,
Condition cond) {
DCHECK(cond == al);
if (CanBeImmediate(index)) {
DecompressTagged(destination, ReadOnlyRootPtr(index));
return;
}
LoadU64(destination,
MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
}
void MacroAssembler::LoadTaggedField(const Register& destination,
const MemOperand& field_operand) {
ASM_CODE_COMMENT(this);
if (COMPRESS_POINTERS_BOOL) {
DecompressTagged(destination, field_operand);
} else {
LoadU64(destination, field_operand);
}
}
void MacroAssembler::LoadTaggedFieldWithoutDecompressing(
const Register& destination, const MemOperand& field_operand) {
if (COMPRESS_POINTERS_BOOL) {
LoadU32(destination, field_operand);
} else {
LoadU64(destination, field_operand);
}
}
void MacroAssembler::SmiUntag(Register dst, const MemOperand& src, RCBit rc) {
if (SmiValuesAre31Bits()) {
LoadU32(dst, src);
} else {
LoadU64(dst, src);
}
SmiUntag(dst, rc);
}
void MacroAssembler::SmiUntagField(Register dst, const MemOperand& src,
RCBit rc) {
SmiUntag(dst, src, rc);
}
void MacroAssembler::StoreTaggedField(const Register& value,
const MemOperand& dst_field_operand) {
if (COMPRESS_POINTERS_BOOL) {
RecordComment("[ StoreTagged");
StoreU32(value, dst_field_operand);
RecordComment("]");
} else {
StoreU64(value, dst_field_operand);
}
}
void MacroAssembler::DecompressTaggedSigned(Register destination,
Register src) {
RecordComment("[ DecompressTaggedSigned");
ZeroExtWord32(destination, src);
RecordComment("]");
}
void MacroAssembler::DecompressTaggedSigned(Register destination,
MemOperand field_operand) {
RecordComment("[ DecompressTaggedSigned");
LoadU32(destination, field_operand);
RecordComment("]");
}
void MacroAssembler::DecompressTagged(Register destination, Register source) {
RecordComment("[ DecompressTagged");
ZeroExtWord32(destination, source);
add(destination, destination, kPtrComprCageBaseRegister);
RecordComment("]");
}
void MacroAssembler::DecompressTagged(Register destination,
MemOperand field_operand) {
RecordComment("[ DecompressTagged");
LoadU32(destination, field_operand);
add(destination, destination, kPtrComprCageBaseRegister);
RecordComment("]");
}
void MacroAssembler::DecompressTagged(const Register& destination,
Tagged_t immediate) {
ASM_CODE_COMMENT(this);
AddS64(destination, kPtrComprCageBaseRegister,
Operand(immediate, RelocInfo::Mode::NO_INFO));
}
void MacroAssembler::LoadTaggedSignedField(Register destination,
MemOperand field_operand) {
if (COMPRESS_POINTERS_BOOL) {
DecompressTaggedSigned(destination, field_operand);
} else {
LoadU64(destination, field_operand);
}
}
void MacroAssembler::RecordWriteField(Register object, int offset,
Register value, Register slot_address,
LinkRegisterStatus lr_status,
SaveFPRegsMode save_fp,
SmiCheck smi_check) {
// First, check if a write barrier is even needed. The tests below
// catch stores of Smis.
Label done;
// Skip barrier if writing a smi.
if (smi_check == SmiCheck::kInline) {
JumpIfSmi(value, &done);
}
// Although the object register is tagged, the offset is relative to the start
// of the object, so so offset must be a multiple of kSystemPointerSize.
DCHECK(IsAligned(offset, kTaggedSize));
AddS64(slot_address, object, Operand(offset - kHeapObjectTag));
if (v8_flags.slow_debug_code) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label ok;
andi(scratch, slot_address, Operand(kTaggedSize - 1));
beq(&ok, cr0);
stop();
bind(&ok);
}
RecordWrite(object, slot_address, value, lr_status, save_fp, SmiCheck::kOmit);
bind(&done);
// Clobber clobbered input registers when running with the debug-code flag
// turned on to provoke errors.
if (v8_flags.slow_debug_code) {
mov(value, Operand(base::bit_cast<intptr_t>(kZapValue + 4)));
mov(slot_address, Operand(base::bit_cast<intptr_t>(kZapValue + 8)));
}
}
void MacroAssembler::Zero(const MemOperand& dest) {
ASM_CODE_COMMENT(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand::Zero());
StoreU64(scratch, dest);
}
void MacroAssembler::Zero(const MemOperand& dest1, const MemOperand& dest2) {
ASM_CODE_COMMENT(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand::Zero());
StoreU64(scratch, dest1);
StoreU64(scratch, dest2);
}
void MacroAssembler::MaybeSaveRegisters(RegList registers) {
if (registers.is_empty()) return;
MultiPush(registers);
}
void MacroAssembler::MaybeRestoreRegisters(RegList registers) {
if (registers.is_empty()) return;
MultiPop(registers);
}
void MacroAssembler::CallEphemeronKeyBarrier(Register object,
Register slot_address,
SaveFPRegsMode fp_mode) {
DCHECK(!AreAliased(object, slot_address));
RegList registers =
WriteBarrierDescriptor::ComputeSavedRegisters(object, slot_address);
MaybeSaveRegisters(registers);
Register object_parameter = WriteBarrierDescriptor::ObjectRegister();
Register slot_address_parameter =
WriteBarrierDescriptor::SlotAddressRegister();
push(object);
push(slot_address);
pop(slot_address_parameter);
pop(object_parameter);
CallBuiltin(Builtins::EphemeronKeyBarrier(fp_mode));
MaybeRestoreRegisters(registers);
}
void MacroAssembler::CallRecordWriteStubSaveRegisters(Register object,
Register slot_address,
SaveFPRegsMode fp_mode,
StubCallMode mode) {
DCHECK(!AreAliased(object, slot_address));
RegList registers =
WriteBarrierDescriptor::ComputeSavedRegisters(object, slot_address);
MaybeSaveRegisters(registers);
Register object_parameter = WriteBarrierDescriptor::ObjectRegister();
Register slot_address_parameter =
WriteBarrierDescriptor::SlotAddressRegister();
push(object);
push(slot_address);
pop(slot_address_parameter);
pop(object_parameter);
CallRecordWriteStub(object_parameter, slot_address_parameter, fp_mode, mode);
MaybeRestoreRegisters(registers);
}
void MacroAssembler::CallRecordWriteStub(Register object, Register slot_address,
SaveFPRegsMode fp_mode,
StubCallMode mode) {
// Use CallRecordWriteStubSaveRegisters if the object and slot registers
// need to be caller saved.
DCHECK_EQ(WriteBarrierDescriptor::ObjectRegister(), object);
DCHECK_EQ(WriteBarrierDescriptor::SlotAddressRegister(), slot_address);
#if V8_ENABLE_WEBASSEMBLY
if (mode == StubCallMode::kCallWasmRuntimeStub) {
// Use {near_call} for direct Wasm call within a module.
auto wasm_target =
static_cast<Address>(wasm::WasmCode::GetRecordWriteBuiltin(fp_mode));
Call(wasm_target, RelocInfo::WASM_STUB_CALL);
#else
if (false) {
#endif
} else {
CallBuiltin(Builtins::RecordWrite(fp_mode), al);
}
}
void MacroAssembler::CallVerifySkippedWriteBarrierStubSaveRegisters(
Register object, Register value, SaveFPRegsMode fp_mode) {
ASM_CODE_COMMENT(this);
PushCallerSaved(fp_mode);
CallVerifySkippedWriteBarrierStub(object, value);
PopCallerSaved(fp_mode);
}
void MacroAssembler::CallVerifySkippedWriteBarrierStub(Register object,
Register value) {
ASM_CODE_COMMENT(this);
push(value);
push(object);
pop(kCArgRegs[0]);
pop(kCArgRegs[1]);
PrepareCallCFunction(2);
CallCFunction(ExternalReference::verify_skipped_write_barrier(), 2);
}
// Will clobber 4 registers: object, address, scratch, ip. The
// register 'object' contains a heap object pointer. The heap object
// tag is shifted away.
void MacroAssembler::RecordWrite(Register object, Register slot_address,
Register value, LinkRegisterStatus lr_status,
SaveFPRegsMode fp_mode, SmiCheck smi_check) {
DCHECK(!AreAliased(object, value, slot_address));
if (v8_flags.slow_debug_code) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadTaggedField(scratch, MemOperand(slot_address));
CmpS64(scratch, value);
Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
}
if (v8_flags.disable_write_barriers) {
return;
}
// First, check if a write barrier is even needed. The tests below
// catch stores of smis and stores into the young generation.
Label done;
if (smi_check == SmiCheck::kInline) {
JumpIfSmi(value, &done);
}
CheckPageFlag(value,
value, // Used as scratch.
MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
CheckPageFlag(object,
value, // Used as scratch.
MemoryChunk::kPointersFromHereAreInterestingMask, eq, &done);
// Record the actual write.
if (lr_status == kLRHasNotBeenSaved) {
PushLR();
}
CallRecordWriteStubSaveRegisters(object, slot_address, fp_mode);
if (lr_status == kLRHasNotBeenSaved) {
PopLR();
}
if (v8_flags.slow_debug_code) mov(slot_address, Operand(kZapValue));
bind(&done);
// Clobber clobbered registers when running with the debug-code flag
// turned on to provoke errors.
if (v8_flags.slow_debug_code) {
mov(slot_address, Operand(base::bit_cast<intptr_t>(kZapValue + 12)));
mov(value, Operand(base::bit_cast<intptr_t>(kZapValue + 16)));
}
}
void MacroAssembler::PushLR(Register scratch) {
UseScratchRegisterScope temps(this);
if (scratch == no_reg) {
scratch = temps.Acquire();
}
mflr(scratch);
push(scratch);
}
void MacroAssembler::PopLR(Register scratch) {
UseScratchRegisterScope temps(this);
if (scratch == no_reg) {
scratch = temps.Acquire();
}
pop(scratch);
mtlr(scratch);
}
void MacroAssembler::PushCommonFrame(Register marker_reg) {
int fp_delta = 0;
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mflr(scratch);
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
if (marker_reg.is_valid()) {
Push(scratch, fp, kConstantPoolRegister, marker_reg);
fp_delta = 2;
} else {
Push(scratch, fp, kConstantPoolRegister);
fp_delta = 1;
}
} else {
if (marker_reg.is_valid()) {
Push(scratch, fp, marker_reg);
fp_delta = 1;
} else {
Push(scratch, fp);
fp_delta = 0;
}
}
addi(fp, sp, Operand(fp_delta * kSystemPointerSize));
}
void MacroAssembler::PushStandardFrame(Register function_reg) {
int fp_delta = 0;
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mflr(scratch);
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
if (function_reg.is_valid()) {
Push(scratch, fp, kConstantPoolRegister, cp, function_reg);
fp_delta = 3;
} else {
Push(scratch, fp, kConstantPoolRegister, cp);
fp_delta = 2;
}
} else {
if (function_reg.is_valid()) {
Push(scratch, fp, cp, function_reg);
fp_delta = 2;
} else {
Push(scratch, fp, cp);
fp_delta = 1;
}
}
addi(fp, sp, Operand(fp_delta * kSystemPointerSize));
Push(kJavaScriptCallArgCountRegister);
}
void MacroAssembler::RestoreFrameStateForTailCall() {
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
LoadU64(kConstantPoolRegister,
MemOperand(fp, StandardFrameConstants::kConstantPoolOffset));
set_constant_pool_available(false);
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadU64(scratch, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
LoadU64(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
mtlr(scratch);
}
void MacroAssembler::CanonicalizeNaN(const DoubleRegister dst,
const DoubleRegister src) {
// Turn potential sNaN into qNaN.
LoadDoubleLiteral(kDoubleRegZero, base::Double(0.0));
fsub(dst, src, kDoubleRegZero);
}
void MacroAssembler::ConvertIntToDouble(Register src, DoubleRegister dst) {
MovIntToDouble(dst, src);
fcfid(dst, dst);
}
void MacroAssembler::ConvertUnsignedIntToDouble(Register src,
DoubleRegister dst) {
MovUnsignedIntToDouble(dst, src);
fcfid(dst, dst);
}
void MacroAssembler::ConvertIntToFloat(Register src, DoubleRegister dst) {
MovIntToDouble(dst, src);
fcfids(dst, dst);
}
void MacroAssembler::ConvertUnsignedIntToFloat(Register src,
DoubleRegister dst) {
MovUnsignedIntToDouble(dst, src);
fcfids(dst, dst);
}
void MacroAssembler::ConvertInt64ToDouble(Register src,
DoubleRegister double_dst) {
MovInt64ToDouble(double_dst, src);
fcfid(double_dst, double_dst);
}
void MacroAssembler::ConvertUnsignedInt64ToFloat(Register src,
DoubleRegister double_dst) {
MovInt64ToDouble(double_dst, src);
fcfidus(double_dst, double_dst);
}
void MacroAssembler::ConvertUnsignedInt64ToDouble(Register src,
DoubleRegister double_dst) {
MovInt64ToDouble(double_dst, src);
fcfidu(double_dst, double_dst);
}
void MacroAssembler::ConvertInt64ToFloat(Register src,
DoubleRegister double_dst) {
MovInt64ToDouble(double_dst, src);
fcfids(double_dst, double_dst);
}
void MacroAssembler::ConvertDoubleToInt64(const DoubleRegister double_input,
const Register dst,
const DoubleRegister double_dst,
FPRoundingMode rounding_mode) {
if (rounding_mode == kRoundToZero) {
fctidz(double_dst, double_input);
} else {
SetRoundingMode(rounding_mode);
fctid(double_dst, double_input);
ResetRoundingMode();
}
MovDoubleToInt64(
dst, double_dst);
}
void MacroAssembler::ConvertDoubleToUnsignedInt64(
const DoubleRegister double_input, const Register dst,
const DoubleRegister double_dst, FPRoundingMode rounding_mode) {
if (rounding_mode == kRoundToZero) {
fctiduz(double_dst, double_input);
} else {
SetRoundingMode(rounding_mode);
fctidu(double_dst, double_input);
ResetRoundingMode();
}
MovDoubleToInt64(dst, double_dst);
}
void MacroAssembler::LoadConstantPoolPointerRegisterFromCodeTargetAddress(
Register code_target_address) {
// Builtins do not use the constant pool (see is_constant_pool_available).
static_assert(InstructionStream::kOnHeapBodyIsContiguous);
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
Register scratch2 = temps.Acquire();
LoadU64(scratch2,
FieldMemOperand(code_target_address, Code::kInstructionStartOffset));
LoadU32(scratch1,
FieldMemOperand(code_target_address, Code::kInstructionSizeOffset));
add(scratch2, scratch1, scratch2);
LoadU32(
kConstantPoolRegister,
FieldMemOperand(code_target_address, Code::kConstantPoolOffsetOffset));
add(kConstantPoolRegister, scratch2, kConstantPoolRegister);
}
void MacroAssembler::LoadPC(Register dst) {
addpcis(dst, Operand(0));
}
void MacroAssembler::ComputeCodeStartAddress(Register dst) {
LoadPC(dst);
subi(dst, dst, Operand(pc_offset()));
}
void MacroAssembler::LoadConstantPoolPointerRegister() {
//
// Builtins do not use the constant pool (see is_constant_pool_available).
static_assert(InstructionStream::kOnHeapBodyIsContiguous);
LoadPC(kConstantPoolRegister);
int32_t delta = -pc_offset();
add_label_offset(kConstantPoolRegister, kConstantPoolRegister,
ConstantPoolPosition(), delta);
}
void MacroAssembler::StubPrologue(StackFrame::Type type) {
{
ConstantPoolUnavailableScope constant_pool_unavailable(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(StackFrame::TypeToMarker(type)));
PushCommonFrame(scratch);
}
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
LoadConstantPoolPointerRegister();
set_constant_pool_available(true);
}
}
void MacroAssembler::Prologue() {
PushStandardFrame(r4);
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
// base contains prologue address
LoadConstantPoolPointerRegister();
set_constant_pool_available(true);
}
}
void MacroAssembler::DropArguments(Register count) { Drop(count); }
void MacroAssembler::DropArgumentsAndPushNewReceiver(Register argc,
Register receiver) {
DCHECK(!AreAliased(argc, receiver));
DropArguments(argc);
push(receiver);
}
void MacroAssembler::EnterFrame(StackFrame::Type type,
bool load_constant_pool_pointer_reg) {
UseScratchRegisterScope temps(this);
if (V8_EMBEDDED_CONSTANT_POOL_BOOL && load_constant_pool_pointer_reg) {
// Push type explicitly so we can leverage the constant pool.
// This path cannot rely on ip containing code entry.
PushCommonFrame();
LoadConstantPoolPointerRegister();
if (!StackFrame::IsJavaScript(type)) {
Register scratch = temps.Acquire();
mov(scratch, Operand(StackFrame::TypeToMarker(type)));
push(scratch);
}
} else {
Register scratch = no_reg;
if (!StackFrame::IsJavaScript(type)) {
scratch = temps.Acquire();
mov(scratch, Operand(StackFrame::TypeToMarker(type)));
}
PushCommonFrame(scratch);
}
#if V8_ENABLE_WEBASSEMBLY
if (type == StackFrame::WASM) Push(kWasmImplicitArgRegister);
#endif // V8_ENABLE_WEBASSEMBLY
}
int MacroAssembler::LeaveFrame(StackFrame::Type type, int stack_adjustment) {
ConstantPoolUnavailableScope constant_pool_unavailable(this);
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
Register scratch2 = temps.Acquire();
// r3: preserved
// r4: preserved
// r5: preserved
// Drop the execution stack down to the frame pointer and restore
// the caller's state.
int frame_ends;
LoadU64(scratch2, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
LoadU64(scratch1, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
LoadU64(kConstantPoolRegister,
MemOperand(fp, StandardFrameConstants::kConstantPoolOffset));
}
mtlr(scratch2);
frame_ends = pc_offset();
AddS64(sp, fp,
Operand(StandardFrameConstants::kCallerSPOffset + stack_adjustment));
mr(fp, scratch1);
return frame_ends;
}
// ExitFrame layout (probably wrongish.. needs updating)
//
// SP -> previousSP
// LK reserved
// sp_on_exit (for debug?)
// oldSP->prev SP
// LK
// <parameters on stack>
// Prior to calling EnterExitFrame, we've got a bunch of parameters
// on the stack that we need to wrap a real frame around.. so first
// we reserve a slot for LK and push the previous SP which is captured
// in the fp register (r31)
// Then - we buy a new frame
void MacroAssembler::EnterExitFrame(int stack_space,
StackFrame::Type frame_type) {
DCHECK(frame_type == StackFrame::EXIT ||
frame_type == StackFrame::BUILTIN_EXIT ||
frame_type == StackFrame::API_NAMED_ACCESSOR_EXIT ||
frame_type == StackFrame::API_CALLBACK_EXIT);
// Set up the frame structure on the stack.
DCHECK_EQ(2 * kSystemPointerSize, ExitFrameConstants::kCallerSPDisplacement);
DCHECK_EQ(1 * kSystemPointerSize, ExitFrameConstants::kCallerPCOffset);
DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
// This is an opportunity to build a frame to wrap
// all of the pushes that have happened inside of V8
// since we were called from C code
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(StackFrame::TypeToMarker(frame_type)));
PushCommonFrame(scratch);
// Reserve room for saved entry sp.
subi(sp, fp, Operand(ExitFrameConstants::kFixedFrameSizeFromFp));
if (v8_flags.debug_code) {
li(r8, Operand::Zero());
StoreU64(r8, MemOperand(fp, ExitFrameConstants::kSPOffset));
}
if (V8_EMBEDDED_CONSTANT_POOL_BOOL) {
StoreU64(kConstantPoolRegister,
MemOperand(fp, ExitFrameConstants::kConstantPoolOffset));
}
// Save the frame pointer and the context in top.
StoreU64(fp, AsMemOperand(IsolateFieldId::kCEntryFP));
StoreU64(cp, AsMemOperand(IsolateFieldId::kContext));
AddS64(sp, sp, Operand(-(stack_space + 1) * kSystemPointerSize));
// Allocate and align the frame preparing for calling the runtime
// function.
const int frame_alignment = ActivationFrameAlignment();
if (frame_alignment > kSystemPointerSize) {
DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
ClearRightImm(sp, sp,
Operand(base::bits::WhichPowerOfTwo(frame_alignment)));
}
li(scratch, Operand::Zero());
StoreU64WithUpdate(scratch, MemOperand(sp, -kNumRequiredStackFrameSlots *
kSystemPointerSize));
// Set the exit frame sp value to point just before the return address
// location.
AddS64(r8, sp, Operand((kStackFrameExtraParamSlot + 1) * kSystemPointerSize));
StoreU64(r8, MemOperand(fp, ExitFrameConstants::kSPOffset));
}
int MacroAssembler::ActivationFrameAlignment() {
#if !defined(USE_SIMULATOR)
// Running on the real platform. Use the alignment as mandated by the local
// environment.
// Note: This will break if we ever start generating snapshots on one PPC
// platform for another PPC platform with a different alignment.
return base::OS::ActivationFrameAlignment();
#else // Simulated
// If we are using the simulator then we should always align to the expected
// alignment. As the simulator is used to generate snapshots we do not know
// if the target platform will need alignment, so this is controlled from a
// flag.
return v8_flags.sim_stack_alignment;
#endif
}
void MacroAssembler::LeaveExitFrame() {
ConstantPoolUnavailableScope constant_pool_unavailable(this);
// Restore current context from top and clear it in debug mode.
LoadU64(cp, AsMemOperand(IsolateFieldId::kContext));
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
#ifdef DEBUG
mov(scratch, Operand(Context::kNoContext));
StoreU64(scratch, AsMemOperand(IsolateFieldId::kContext));
#endif
// Clear the top frame.
mov(scratch, Operand::Zero());
StoreU64(scratch, AsMemOperand(IsolateFieldId::kCEntryFP));
// Tear down the exit frame, pop the arguments, and return.
LeaveFrame(StackFrame::EXIT);
}
void MacroAssembler::MovFromFloatResult(const DoubleRegister dst) {
Move(dst, d1);
}
void MacroAssembler::MovFromFloatParameter(const DoubleRegister dst) {
Move(dst, d1);
}
void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
DCHECK(root_array_available());
intptr_t offset = kind == StackLimitKind::kRealStackLimit
? IsolateData::real_jslimit_offset()
: IsolateData::jslimit_offset();
CHECK(is_int32(offset));
LoadU64(destination, MemOperand(kRootRegister, offset));
}
void MacroAssembler::StackOverflowCheck(Register num_args,
Label* stack_overflow) {
// Check the stack for overflow. We are not trying to catch
// interruptions (e.g. debug break and preemption) here, so the "real stack
// limit" is checked.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadStackLimit(scratch, StackLimitKind::kRealStackLimit);
// Make scratch the space we have left. The stack might already be overflowed
// here which will cause scratch to become negative.
sub(scratch, sp, scratch);
// Check if the arguments will overflow the stack.
Register bytes_needed = temps.Acquire();
ShiftLeftU64(bytes_needed, num_args, Operand(kSystemPointerSizeLog2));
CmpS64(scratch, bytes_needed);
ble(stack_overflow); // Signed comparison.
}
void MacroAssembler::InvokePrologue(Register expected_parameter_count,
Register actual_parameter_count,
InvokeType type) {
Label regular_invoke;
// r3: actual arguments count
// r4: function (passed through to callee)
// r5: expected arguments count
DCHECK_EQ(actual_parameter_count, r3);
DCHECK_EQ(expected_parameter_count, r5);
// If overapplication or if the actual argument count is equal to the
// formal parameter count, no need to push extra undefined values.
sub(expected_parameter_count, expected_parameter_count,
actual_parameter_count, LeaveOE, SetRC);
ble(®ular_invoke, cr0);
Label stack_overflow;
Register scratch = r7;
StackOverflowCheck(expected_parameter_count, &stack_overflow);
// Underapplication. Move the arguments already in the stack, including the
// receiver and the return address.
{
Label copy, skip;
Register src = r9, dest = r8;
addi(src, sp, Operand(-kSystemPointerSize));
ShiftLeftU64(scratch, expected_parameter_count,
Operand(kSystemPointerSizeLog2));
sub(sp, sp, scratch);
// Update stack pointer.
addi(dest, sp, Operand(-kSystemPointerSize));
mr(scratch, actual_parameter_count);
cmpi(scratch, Operand::Zero());
ble(&skip);
mtctr(scratch);
bind(©);
LoadU64WithUpdate(scratch, MemOperand(src, kSystemPointerSize));
StoreU64WithUpdate(scratch, MemOperand(dest, kSystemPointerSize));
bdnz(©);
bind(&skip);
}
// Fill remaining expected arguments with undefined values.
LoadRoot(scratch, RootIndex::kUndefinedValue);
{
mtctr(expected_parameter_count);
Label loop;
bind(&loop);
StoreU64WithUpdate(scratch, MemOperand(r8, kSystemPointerSize));
bdnz(&loop);
}
b(®ular_invoke);
bind(&stack_overflow);
{
FrameScope frame(
this, has_frame() ? StackFrame::NO_FRAME_TYPE : StackFrame::INTERNAL);
CallRuntime(Runtime::kThrowStackOverflow);
bkpt(0);
}
bind(®ular_invoke);
}
void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
Register expected_parameter_count,
Register actual_parameter_count) {
Label skip_hook;
ExternalReference debug_hook_active =
ExternalReference::debug_hook_on_function_call_address(isolate());
Move(r7, debug_hook_active);
LoadU8(r7, MemOperand(r7));
extsb(r7, r7);
CmpSmiLiteral(r7, Smi::zero());
beq(&skip_hook);
{
// Load receiver to pass it later to DebugOnFunctionCall hook.
LoadReceiver(r7);
FrameScope frame(
this, has_frame() ? StackFrame::NO_FRAME_TYPE : StackFrame::INTERNAL);
SmiTag(expected_parameter_count);
Push(expected_parameter_count);
SmiTag(actual_parameter_count);
Push(actual_parameter_count);
if (new_target.is_valid()) {
Push(new_target);
}
Push(fun, fun, r7);
CallRuntime(Runtime::kDebugOnFunctionCall);
Pop(fun);
if (new_target.is_valid()) {
Pop(new_target);
}
Pop(actual_parameter_count);
SmiUntag(actual_parameter_count);
Pop(expected_parameter_count);
SmiUntag(expected_parameter_count);
}
bind(&skip_hook);
}
void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
Register expected_parameter_count,
Register actual_parameter_count,
InvokeType type) {
// You can't call a function without a valid frame.
DCHECK_IMPLIES(type == InvokeType::kCall, has_frame());
DCHECK_EQ(function, r4);
DCHECK_IMPLIES(new_target.is_valid(), new_target == r6);
// On function call, call into the debugger if necessary.
CheckDebugHook(function, new_target, expected_parameter_count,
actual_parameter_count);
// Clear the new.target register if not given.
if (!new_target.is_valid()) {
LoadRoot(r6, RootIndex::kUndefinedValue);
}
InvokePrologue(expected_parameter_count, actual_parameter_count, type);
// We call indirectly through the code field in the function to
// allow recompilation to take effect without changing any of the
// call sites.
constexpr int unused_argument_count = 0;
switch (type) {
case InvokeType::kCall:
CallJSFunction(function, unused_argument_count);
break;
case InvokeType::kJump:
JumpJSFunction(function);
break;
}
}
void MacroAssembler::InvokeFunctionWithNewTarget(
Register fun, Register new_target, Register actual_parameter_count,
InvokeType type) {
// You can't call a function without a valid frame.
DCHECK_IMPLIES(type == InvokeType::kCall, has_frame());
// Contract with called JS functions requires that function is passed in r4.
DCHECK_EQ(fun, r4);
Register expected_reg = r5;
Register temp_reg = r7;
LoadTaggedField(
temp_reg,
FieldMemOperand(r4, offsetof(JSFunction, shared_function_info_)));
LoadTaggedField(cp, FieldMemOperand(r4, offsetof(JSFunction, context_)));
LoadU16(expected_reg,
FieldMemOperand(
temp_reg, offsetof(SharedFunctionInfo, formal_parameter_count_)));
InvokeFunctionCode(fun, new_target, expected_reg, actual_parameter_count,
type);
}
void MacroAssembler::InvokeFunction(Register function,
Register expected_parameter_count,
Register actual_parameter_count,
InvokeType type) {
// You can't call a function without a valid frame.
DCHECK_IMPLIES(type == InvokeType::kCall, has_frame());
// Contract with called JS functions requires that function is passed in r4.
DCHECK_EQ(function, r4);
// Get the function and setup the context.
LoadTaggedField(cp, FieldMemOperand(r4, offsetof(JSFunction, context_)));
InvokeFunctionCode(r4, no_reg, expected_parameter_count,
actual_parameter_count, type);
}
void MacroAssembler::PushStackHandler() {
// Adjust this code if not the case.
static_assert(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
static_assert(StackHandlerConstants::kNextOffset == 0 * kSystemPointerSize);
Push(Smi::zero()); // Padding.
// Link the current handler as the next handler.
// Preserve r4-r8.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadU64(scratch, AsMemOperand(IsolateFieldId::kHandler));
push(scratch);
// Set this new handler as the current one.
StoreU64(sp, AsMemOperand(IsolateFieldId::kHandler));
}
void MacroAssembler::PopStackHandler() {
static_assert(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
static_assert(StackHandlerConstants::kNextOffset == 0);
pop(r4);
StoreU64(r4, AsMemOperand(IsolateFieldId::kHandler));
Drop(1); // Drop padding.
}
// Sets equality condition flags.
void MacroAssembler::IsObjectType(Register object, Register scratch1,
Register scratch2, InstanceType type) {
ASM_CODE_COMMENT(this);
CompareObjectType(object, scratch1, scratch2, type);
}
void MacroAssembler::CompareObjectTypeRange(Register object, Register map,
Register type_reg,
InstanceType lower_limit,
InstanceType upper_limit) {
ASM_CODE_COMMENT(this);
LoadMap(map, object);
CompareInstanceTypeRange(map, type_reg, lower_limit, upper_limit);
}
void MacroAssembler::CompareRange(Register value, unsigned lower_limit,
unsigned higher_limit) {
ASM_CODE_COMMENT(this);
DCHECK_LT(lower_limit, higher_limit);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (lower_limit != 0) {
mov(scratch, Operand(lower_limit));
sub(scratch, value, scratch);
cmpli(scratch, Operand(higher_limit - lower_limit));
} else {
mov(scratch, Operand(higher_limit));
CmpU64(value, scratch);
}
}
void MacroAssembler::CompareInstanceTypeRange(Register map, Register type_reg,
InstanceType lower_limit,
InstanceType higher_limit) {
DCHECK_LT(lower_limit, higher_limit);
LoadU16(type_reg, FieldMemOperand(map, offsetof(Map, instance_type_)));
CompareRange(type_reg, lower_limit, higher_limit);
}
void MacroAssembler::CompareTaggedRoot(const Register& obj, RootIndex index) {
ASM_CODE_COMMENT(this);
// Some smi roots contain system pointer size values like stack limits.
DCHECK(base::IsInRange(index, RootIndex::kFirstStrongOrReadOnlyRoot,
RootIndex::kLastStrongOrReadOnlyRoot));
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadRoot(scratch, index);
CompareTagged(obj, scratch);
}
void MacroAssembler::CompareRoot(Register obj, RootIndex index) {
ASM_CODE_COMMENT(this);
if (!base::IsInRange(index, RootIndex::kFirstStrongOrReadOnlyRoot,
RootIndex::kLastStrongOrReadOnlyRoot)) {
// Some smi roots contain system pointer size values like stack limits.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadRoot(scratch, index);
CmpU64(obj, scratch);
return;
}
CompareTaggedRoot(obj, index);
}
void MacroAssembler::MinF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, DoubleRegister scratch) {
Label return_nan, done;
fcmpu(lhs, rhs);
bunordered(&return_nan);
xsmindp(dst, lhs, rhs);
b(&done);
bind(&return_nan);
/* If left or right are NaN, fadd propagates the appropriate one.*/
fadd(dst, lhs, rhs);
bind(&done);
}
void MacroAssembler::MaxF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, DoubleRegister scratch) {
Label return_nan, done;
fcmpu(lhs, rhs);
bunordered(&return_nan);
xsmaxdp(dst, lhs, rhs);
b(&done);
bind(&return_nan);
/* If left or right are NaN, fadd propagates the appropriate one.*/
fadd(dst, lhs, rhs);
bind(&done);
}
void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
unsigned higher_limit,
Label* on_in_range) {
CompareRange(value, lower_limit, higher_limit);
ble(on_in_range);
}
void MacroAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
Register result,
DoubleRegister double_input,
StubCallMode stub_mode,
DoubleRegister double_scratch) {
Label done;
TryInlineTruncateDoubleToI(result, double_input, &done, double_scratch);
// If we fell through then inline version didn't succeed - call stub instead.
PushLR();
// Put input on stack.
stfdu(double_input, MemOperand(sp, -kDoubleSize));
#if V8_ENABLE_WEBASSEMBLY
if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
Call(static_cast<Address>(Builtin::kDoubleToI), RelocInfo::WASM_STUB_CALL);
#else
// For balance.
if (false) {
#endif // V8_ENABLE_WEBASSEMBLY
} else {
CallBuiltin(Builtin::kDoubleToI);
}
LoadU64(result, MemOperand(sp));
addi(sp, sp, Operand(kDoubleSize));
PopLR();
bind(&done);
}
void MacroAssembler::TryInlineTruncateDoubleToI(Register result,
DoubleRegister double_input,
Label* done,
DoubleRegister double_scratch) {
CHECK(!AreAliased(double_scratch, double_input));
ConvertDoubleToInt64(double_input,
result, double_scratch);
// Test for overflow
TestIfInt32(result);
beq(done);
}
#ifdef V8_ENABLE_DEBUG_CODE
void MacroAssembler::AssertFeedbackCell(Register object, Register scratch) {
if (v8_flags.debug_code) {
CompareObjectType(object, scratch, scratch, FEEDBACK_CELL_TYPE);
Assert(eq, AbortReason::kExpectedFeedbackCell);
}
}
void MacroAssembler::AssertFeedbackVector(Register object, Register scratch) {
if (v8_flags.debug_code) {
CompareObjectType(object, scratch, scratch, FEEDBACK_VECTOR_TYPE);
Assert(eq, AbortReason::kExpectedFeedbackVector);
}
}
#endif // V8_ENABLE_DEBUG_CODE
void MacroAssembler::GenerateTailCallToReturnedCode(
Runtime::FunctionId function_id) {
// ----------- S t a t e -------------
// -- r3 : actual argument count
// -- r4 : target function (preserved for callee)
// -- r6 : new target (preserved for callee)
// -----------------------------------
{
FrameAndConstantPoolScope scope(this, StackFrame::INTERNAL);
// Push a copy of the target function, the new target and the actual
// argument count.
// Push function as parameter to the runtime call.
SmiTag(kJavaScriptCallArgCountRegister);
Push(kJavaScriptCallTargetRegister, kJavaScriptCallNewTargetRegister,
kJavaScriptCallArgCountRegister, kJavaScriptCallTargetRegister);
CallRuntime(function_id, 1);
// Restore target function, new target and actual argument count.
Pop(kJavaScriptCallTargetRegister, kJavaScriptCallNewTargetRegister,
kJavaScriptCallArgCountRegister);
SmiUntag(kJavaScriptCallArgCountRegister);
}
static_assert(kJavaScriptCallCodeStartRegister == r5, "ABI mismatch");
JumpJSFunction(kJavaScriptCallTargetRegister);
}
void MacroAssembler::CallRuntime(const Runtime::Function* f,
int num_arguments) {
// All parameters are on the stack. r3 has the return value after call.
// If the expected number of arguments of the runtime function is
// constant, we check that the actual number of arguments match the
// expectation.
CHECK(f->nargs < 0 || f->nargs == num_arguments);
// TODO(1236192): Most runtime routines don't need the number of
// arguments passed in because it is constant. At some point we
// should remove this need and make the runtime routine entry code
// smarter.
mov(r3, Operand(num_arguments));
Move(r4, ExternalReference::Create(f));
bool switch_to_central_stack = options().is_wasm;
CallBuiltin(Builtins::RuntimeCEntry(f->result_size, switch_to_central_stack));
}
void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
const Runtime::Function* function = Runtime::FunctionForId(fid);
DCHECK_EQ(1, function->result_size);
if (function->nargs >= 0) {
mov(r3, Operand(function->nargs));
}
JumpToExternalReference(ExternalReference::Create(fid));
}
void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
bool builtin_exit_frame) {
Move(r4, builtin);
TailCallBuiltin(Builtins::CEntry(1, ArgvMode::kStack, builtin_exit_frame));
}
void MacroAssembler::LoadWeakValue(Register out, Register in,
Label* target_if_cleared) {
CmpS32(in, Operand(kClearedWeakHeapObjectLower32));
beq(target_if_cleared);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(~kWeakHeapObjectMask));
and_(out, in, scratch);
}
void MacroAssembler::EmitIncrementCounter(StatsCounter* counter, int value,
Register scratch1,
Register scratch2) {
DCHECK_GT(value, 0);
if (v8_flags.native_code_counters && counter->Enabled()) {
// This operation has to be exactly 32-bit wide in case the external
// reference table redirects the counter to a uint32_t dummy_stats_counter_
// field.
Move(scratch2, ExternalReference::Create(counter));
lwz(scratch1, MemOperand(scratch2));
addi(scratch1, scratch1, Operand(value));
stw(scratch1, MemOperand(scratch2));
}
}
void MacroAssembler::EmitDecrementCounter(StatsCounter* counter, int value,
Register scratch1,
Register scratch2) {
DCHECK_GT(value, 0);
if (v8_flags.native_code_counters && counter->Enabled()) {
// This operation has to be exactly 32-bit wide in case the external
// reference table redirects the counter to a uint32_t dummy_stats_counter_
// field.
Move(scratch2, ExternalReference::Create(counter));
lwz(scratch1, MemOperand(scratch2));
subi(scratch1, scratch1, Operand(value));
stw(scratch1, MemOperand(scratch2));
}
}
void MacroAssembler::Check(Condition cond, AbortReason reason, CRegister cr) {
Label L;
b(to_condition(cond), &L, cr);
Abort(reason);
// will not return here
bind(&L);
}
void MacroAssembler::Abort(AbortReason reason) {
ASM_CODE_COMMENT(this);
if (v8_flags.code_comments) {
RecordComment("Abort message:", SourceLocation{});
RecordComment(GetAbortReason(reason), SourceLocation{});
}
// Without debug code, save the code size and just trap.
if (!v8_flags.debug_code || v8_flags.trap_on_abort) {
stop();
return;
}
if (should_abort_hard()) {
// We don't care if we constructed a frame. Just pretend we did.
FrameScope assume_frame(this, StackFrame::NO_FRAME_TYPE);
mov(r3, Operand(static_cast<int>(reason)));
PrepareCallCFunction(1);
Register dst = ip;
if (!ABI_CALL_VIA_IP) {
dst = r4;
}
Move(dst, ExternalReference::abort_with_reason());
// Use Call directly to avoid any unneeded overhead. The function won't
// return anyway.
Call(dst);
return;
}
LoadSmiLiteral(r4, Smi::FromInt(static_cast<int>(reason)));
{
// We don't actually want to generate a pile of code for this, so just
// claim there is a stack frame, without generating one.
FrameScope scope(this, StackFrame::NO_FRAME_TYPE);
if (root_array_available()) {
// Generate an indirect call via builtins entry table here in order to
// ensure that the interpreter_entry_return_pc_offset is the same for
// InterpreterEntryTrampoline and InterpreterEntryTrampolineForProfiling
// when v8_flags.debug_code is enabled.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadEntryFromBuiltin(Builtin::kAbort, scratch);
Call(scratch);
} else {
CallBuiltin(Builtin::kAbort);
}
}
// will not return here
}
void MacroAssembler::LoadMap(Register destination, Register object) {
LoadTaggedField(destination,
FieldMemOperand(object, offsetof(HeapObject, map_)));
}
void MacroAssembler::LoadFeedbackCell(Register dst, Register closure) {
LoadTaggedField(
dst, FieldMemOperand(closure, offsetof(JSFunction, feedback_cell_)));
}
void MacroAssembler::LoadFeedbackVectorFromCell(Register dst,
Register feedback_cell,
Register scratch,
Label* fbv_undef) {
Label done;
LoadTaggedField(
dst, FieldMemOperand(feedback_cell, offsetof(FeedbackCell, value_)));
// Check if feedback vector is valid.
LoadTaggedField(scratch, FieldMemOperand(dst, offsetof(HeapObject, map_)));
LoadU16(scratch, FieldMemOperand(scratch, offsetof(Map, instance_type_)));
CmpS32(scratch, Operand(FEEDBACK_VECTOR_TYPE));
b(eq, &done);
// Not valid, load undefined.
LoadRoot(dst, RootIndex::kUndefinedValue);
b(fbv_undef);
bind(&done);
}
void MacroAssembler::LoadFeedbackVector(Register dst, Register closure,
Register scratch, Label* fbv_undef) {
LoadFeedbackCell(dst, closure);
LoadFeedbackVectorFromCell(dst, dst, scratch, fbv_undef);
}
void MacroAssembler::LoadInterpreterDataBytecodeArray(
Register destination, Register interpreter_data) {
LoadTaggedField(destination,
FieldMemOperand(interpreter_data,
offsetof(InterpreterData, bytecode_array_)));
}
void MacroAssembler::LoadInterpreterDataInterpreterTrampoline(
Register destination, Register interpreter_data) {
LoadTaggedField(
destination,
FieldMemOperand(interpreter_data,
offsetof(InterpreterData, interpreter_trampoline_)));
}
void MacroAssembler::LoadCompressedMap(Register dst, Register object) {
ASM_CODE_COMMENT(this);
LoadU32(dst, FieldMemOperand(object, offsetof(HeapObject, map_)));
}
void MacroAssembler::LoadNativeContextSlot(Register dst, int index) {
LoadMap(dst, cp);
LoadTaggedField(
dst,
FieldMemOperand(
dst, offsetof(Map, constructor_or_back_pointer_or_native_context_)));
LoadTaggedField(dst, MemOperand(dst, Context::SlotOffset(index)));
}
#ifdef V8_ENABLE_DEBUG_CODE
void MacroAssembler::Assert(Condition cond, AbortReason reason, CRegister cr) {
if (v8_flags.debug_code) Check(cond, reason, cr);
}
void MacroAssembler::AssertUnreachable(AbortReason reason) {
if (v8_flags.debug_code) Abort(reason);
}
void MacroAssembler::AssertZeroExtended(Register int32_register) {
if (!v8_flags.debug_code) return;
ASM_CODE_COMMENT(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, Operand(kMaxUInt32));
CmpS64(int32_register, scratch);
Check(le, AbortReason::k32BitValueInRegisterIsNotZeroExtended);
}
void MacroAssembler::AssertMap(Register object) {
if (!v8_flags.debug_code) return;
ASM_CODE_COMMENT(this);
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsNotAMap);
UseScratchRegisterScope temps(this);
Register temp = temps.Acquire();
CompareObjectType(object, temp, temp, MAP_TYPE);
Check(eq, AbortReason::kOperandIsNotAMap);
}
void MacroAssembler::AssertNotSmi(Register object) {
if (v8_flags.debug_code) {
static_assert(kSmiTag == 0);
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsASmi, cr0);
}
}
void MacroAssembler::AssertSmi(Register object) {
if (v8_flags.debug_code) {
static_assert(kSmiTag == 0);
TestIfSmi(object);
Check(eq, AbortReason::kOperandIsNotASmi, cr0);
}
}
void MacroAssembler::AssertConstructor(Register object) {
if (v8_flags.debug_code) {
static_assert(kSmiTag == 0);
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor, cr0);
push(object);
LoadMap(object, object);
lbz(object, FieldMemOperand(object, offsetof(Map, bit_field_)));
andi(object, object, Operand(Map::Bits1::IsConstructorBit::kMask));
pop(object);
Check(ne, AbortReason::kOperandIsNotAConstructor, cr0);
}
}
void MacroAssembler::AssertFunction(Register object) {
if (v8_flags.debug_code) {
static_assert(kSmiTag == 0);
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, cr0);
UseScratchRegisterScope temps(this);
Register temp = temps.Acquire();
LoadMap(temp, object);
CompareInstanceTypeRange(temp, temp, FIRST_JS_FUNCTION_TYPE,
LAST_JS_FUNCTION_TYPE);
Check(le, AbortReason::kOperandIsNotAFunction);
}
}
void MacroAssembler::AssertCallableFunction(Register object) {
if (!v8_flags.debug_code) return;
ASM_CODE_COMMENT(this);
static_assert(kSmiTag == 0);
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, cr0);
UseScratchRegisterScope temps(this);
Register temp = temps.Acquire();
LoadMap(temp, object);
CompareInstanceTypeRange(temp, temp, FIRST_CALLABLE_JS_FUNCTION_TYPE,
LAST_CALLABLE_JS_FUNCTION_TYPE);
Check(le, AbortReason::kOperandIsNotACallableFunction);
}
void MacroAssembler::AssertBoundFunction(Register object) {
if (v8_flags.debug_code) {
static_assert(kSmiTag == 0);
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction, cr0);
UseScratchRegisterScope temps(this);
Register temp = temps.Acquire();
CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
Check(eq, AbortReason::kOperandIsNotABoundFunction);
}
}
void MacroAssembler::AssertGeneratorObject(Register object) {
if (!v8_flags.debug_code) return;
TestIfSmi(object);
Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject, cr0);
// Check if JSGeneratorObject
UseScratchRegisterScope temps(this);
Register temp = temps.Acquire();
LoadMap(temp, object);
CompareInstanceTypeRange(temp, temp, FIRST_JS_GENERATOR_OBJECT_TYPE,
LAST_JS_GENERATOR_OBJECT_TYPE);
Check(le, AbortReason::kOperandIsNotAGeneratorObject);
}
void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
if (v8_flags.debug_code) {
Label done_checking;
AssertNotSmi(object);
CompareRoot(object, RootIndex::kUndefinedValue);
beq(&done_checking);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadMap(scratch, object);
CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
Assert(eq, AbortReason::kExpectedUndefinedOrCell);
bind(&done_checking);
}
}
void MacroAssembler::AssertJSAny(Register object, AbortReason abort_reason) {
if (!v8_flags.debug_code) return;
ASM_CODE_COMMENT(this);
UseScratchRegisterScope temps(this);
Register map_tmp = temps.Acquire();
Register tmp = temps.Acquire();
Label ok;
JumpIfSmi(object, &ok);
LoadMap(map_tmp, object);
CompareInstanceType(map_tmp, tmp, LAST_NAME_TYPE);
ble(&ok);
CompareInstanceType(map_tmp, tmp, FIRST_JS_RECEIVER_TYPE);
bge(&ok);
CompareRoot(map_tmp, RootIndex::kHeapNumberMap);
beq(&ok);
CompareRoot(map_tmp, RootIndex::kBigIntMap);
beq(&ok);
CompareRoot(object, RootIndex::kUndefinedValue);
beq(&ok);
CompareRoot(object, RootIndex::kTrueValue);
beq(&ok);
CompareRoot(object, RootIndex::kFalseValue);
beq(&ok);
CompareRoot(object, RootIndex::kNullValue);
beq(&ok);
Abort(abort_reason);
bind(&ok);
}
#endif // V8_ENABLE_DEBUG_CODE
int MacroAssembler::CalculateStackPassedWords(int num_reg_arguments,
int num_double_arguments) {
int stack_passed_words = 0;
if (num_double_arguments > DoubleRegister::kNumRegisters) {
stack_passed_words +=
2 * (num_double_arguments - DoubleRegister::kNumRegisters);
}
// Up to 8 simple arguments are passed in registers r3..r10.
if (num_reg_arguments > kRegisterPassedArguments) {
stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
}
return stack_passed_words;
}
void MacroAssembler::PrepareCallCFunction(int num_reg_arguments,
int num_double_arguments) {
int frame_alignment = ActivationFrameAlignment();
int stack_passed_arguments =
CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
int stack_space = kNumRequiredStackFrameSlots;
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (frame_alignment > kSystemPointerSize) {
// Make stack end at alignment and make room for stack arguments
// -- preserving original value of sp.
mr(scratch, sp);
AddS64(sp, sp, Operand(-(stack_passed_arguments + 1) * kSystemPointerSize));
DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
ClearRightImm(sp, sp,
Operand(base::bits::WhichPowerOfTwo(frame_alignment)));
StoreU64(scratch,
MemOperand(sp, stack_passed_arguments * kSystemPointerSize));
} else {
// Make room for stack arguments
stack_space += stack_passed_arguments;
}
// Allocate frame with required slots to make ABI work.
li(scratch, Operand::Zero());
StoreU64WithUpdate(scratch,
MemOperand(sp, -stack_space * kSystemPointerSize));
}
void MacroAssembler::MovToFloatParameter(DoubleRegister src) { Move(d1, src); }
void MacroAssembler::MovToFloatResult(DoubleRegister src) { Move(d1, src); }
void MacroAssembler::MovToFloatParameters(DoubleRegister src1,
DoubleRegister src2) {
if (src2 == d1) {
DCHECK(src1 != d2);
Move(d2, src2);
Move(d1, src1);
} else {
Move(d1, src1);
Move(d2, src2);
}
}
int MacroAssembler::CallCFunction(ExternalReference function,
int num_reg_arguments,
int num_double_arguments,
SetIsolateDataSlots set_isolate_data_slots,
bool has_function_descriptor,
Label* return_label) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Move(scratch, function);
return CallCFunction(scratch, num_reg_arguments, num_double_arguments,
set_isolate_data_slots, has_function_descriptor,
return_label);
}
int MacroAssembler::CallCFunction(Register function, int num_reg_arguments,
int num_double_arguments,
SetIsolateDataSlots set_isolate_data_slots,
bool has_function_descriptor,
Label* return_label) {
ASM_CODE_COMMENT(this);
DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
DCHECK(has_frame());
Label get_pc;
if (set_isolate_data_slots == SetIsolateDataSlots::kYes) {
// Save the frame pointer and PC so that the stack layout remains iterable,
// even without an ExitFrame which normally exists between JS and C frames.
DCHECK(!AreAliased(r0, r26, function));
GetLabelAddress(r0, &get_pc);
CHECK(root_array_available());
StoreU64(r0,
ExternalReferenceAsOperand(IsolateFieldId::kFastCCallCallerPC));
StoreU64(fp,
ExternalReferenceAsOperand(IsolateFieldId::kFastCCallCallerFP));
}
Register dest = function;
if (ABI_USES_FUNCTION_DESCRIPTORS && has_function_descriptor) {
// AIX/PPC64BE Linux uses a function descriptor. When calling C code be
// aware of this descriptor and pick up values from it
LoadU64(ToRegister(ABI_TOC_REGISTER),
MemOperand(function, kSystemPointerSize));
LoadU64(ip, MemOperand(function, 0));
dest = ip;
} else if (ABI_CALL_VIA_IP && function != ip) {
// pLinux and Simulator, not AIX
Move(ip, function);
dest = ip;
}
int call_pc_offset;
{
ConstantPoolUnavailableScope block_const_pool_scope(this);
BlockTrampolinePoolScope block_trampoline_pool(this);
Call(dest);
call_pc_offset = pc_offset();
bind(&get_pc);
if (return_label) bind(return_label);
int before_offset = pc_offset();
int stack_passed_arguments =
CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
int stack_space = kNumRequiredStackFrameSlots + stack_passed_arguments;
if (ActivationFrameAlignment() > kSystemPointerSize) {
// Using prefixed load may emit `nop` which then fails the check below.
ld(sp, MemOperand(sp, stack_space * kSystemPointerSize));
} else {
AddS64(sp, sp, Operand(stack_space * kSystemPointerSize));
}
// We assume that the move instruction uses kMaxSizeOfMoveAfterFastCall
// bytes. When we patch in the deopt trampoline, we patch it in after the
// move instruction, so that the stack has been restored correctly.
CHECK_EQ(kMaxSizeOfMoveAfterFastCall, pc_offset() - before_offset);
}
if (set_isolate_data_slots == SetIsolateDataSlots::kYes) {
// We don't unset the PC; the FP is the source of truth.
Register zero_scratch = r0;
mov(zero_scratch, Operand::Zero());
StoreU64(zero_scratch,
ExternalReferenceAsOperand(IsolateFieldId::kFastCCallCallerFP));
}
return call_pc_offset;
}
int MacroAssembler::CallCFunction(ExternalReference function, int num_arguments,
SetIsolateDataSlots set_isolate_data_slots,
bool has_function_descriptor,
Label* return_label) {
return CallCFunction(function, num_arguments, 0, set_isolate_data_slots,
has_function_descriptor, return_label);
}
int MacroAssembler::CallCFunction(Register function, int num_arguments,
SetIsolateDataSlots set_isolate_data_slots,
bool has_function_descriptor,
Label* return_label) {
return CallCFunction(function, num_arguments, 0, set_isolate_data_slots,
has_function_descriptor, return_label);
}
void MacroAssembler::CheckPageFlag(
Register object,
Register scratch, // scratch may be same register as object
int mask, Condition cc, Label* condition_met) {
DCHECK(cc == ne || cc == eq);
ClearRightImm(scratch, object, Operand(kPageSizeBits));
LoadU64(scratch, MemOperand(scratch, MemoryChunk::FlagsOffset()));
UseScratchRegisterScope temps(this);
Register temp = temps.Acquire();
DCHECK_NE(scratch, temp);
mov(temp, Operand(mask));
and_(temp, scratch, temp, SetRC);
if (cc == ne) {
bne(condition_met, cr0);
}
if (cc == eq) {
beq(condition_met, cr0);
}
}
void MacroAssembler::SetRoundingMode(FPRoundingMode RN) { mtfsfi(7, RN); }
void MacroAssembler::ResetRoundingMode() {
mtfsfi(7, kRoundToNearest); // reset (default is kRoundToNearest)
}
////////////////////////////////////////////////////////////////////////////////
//
// New MacroAssembler Interfaces added for PPC
//
////////////////////////////////////////////////////////////////////////////////
void MacroAssembler::LoadIntLiteral(Register dst, int value) {
mov(dst, Operand(value));
}
void MacroAssembler::LoadSmiLiteral(Register dst, Tagged<Smi> smi) {
mov(dst, Operand(smi));
}
void MacroAssembler::LoadDoubleLiteral(DoubleRegister result,
base::Double value) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (V8_EMBEDDED_CONSTANT_POOL_BOOL && is_constant_pool_available() &&
!(scratch == r0 && ConstantPoolAccessIsInOverflow())) {
ConstantPoolEntry::Access access = ConstantPoolAddEntry(value);
if (access == ConstantPoolEntry::OVERFLOWED) {
addis(scratch, kConstantPoolRegister, Operand::Zero());
lfd(result, MemOperand(scratch, 0));
} else {
lfd(result, MemOperand(kConstantPoolRegister, 0));
}
return;
}
// avoid gcc strict aliasing error using union cast
union {
uint64_t dval;
intptr_t ival;
} litVal;
litVal.dval = value.AsUint64();
mov(scratch, Operand(litVal.ival));
mtfprd(result, scratch);
}
void MacroAssembler::MovIntToDouble(DoubleRegister dst, Register src) {
// sign-extend src to 64-bit
mtfprwa(dst, src);
}
void MacroAssembler::MovUnsignedIntToDouble(DoubleRegister dst, Register src) {
// zero-extend src to 64-bit
mtfprwz(dst, src);
}
void MacroAssembler::MovInt64ToDouble(DoubleRegister dst,
Register src) {
mtfprd(dst, src);
}
void MacroAssembler::MovInt64ComponentsToDouble(DoubleRegister dst,
Register src_hi,
Register src_lo) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ShiftLeftU64(scratch, src_hi, Operand(32));
rldimi(scratch, src_lo, 0, 32);
mtfprd(dst, scratch);
}
void MacroAssembler::InsertDoubleLow(DoubleRegister dst, Register src) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mffprd(scratch, dst);
rldimi(scratch, src, 0, 32);
mtfprd(dst, scratch);
}
void MacroAssembler::InsertDoubleHigh(DoubleRegister dst, Register src) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mffprd(scratch, dst);
rldimi(scratch, src, 32, 0);
mtfprd(dst, scratch);
}
void MacroAssembler::MovDoubleLowToInt(Register dst, DoubleRegister src) {
mffprwz(dst, src);
}
void MacroAssembler::MovDoubleHighToInt(Register dst, DoubleRegister src) {
mffprd(dst, src);
srdi(dst, dst, Operand(32));
}
void MacroAssembler::MovDoubleToInt64(Register dst, DoubleRegister src) {
mffprd(dst, src);
}
void MacroAssembler::MovIntToFloat(DoubleRegister dst, Register src) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ShiftLeftU64(scratch, src, Operand(32));
mtfprd(dst, scratch);
xscvspdpn(dst, dst);
}
void MacroAssembler::MovFloatToInt(Register dst, DoubleRegister src,
DoubleRegister scratch) {
xscvdpspn(scratch, src);
mffprwz(dst, scratch);
}
void MacroAssembler::AddS64(Register dst, Register src, Register value, OEBit s,
RCBit r) {
add(dst, src, value, s, r);
}
void MacroAssembler::AddS64(Register dst, Register src, const Operand& value,
OEBit s, RCBit r) {
if (is_int16(value.immediate()) && s == LeaveOE && r == LeaveRC) {
addi(dst, src, value);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, value);
add(dst, src, scratch, s, r);
}
}
void MacroAssembler::SubS64(Register dst, Register src, Register value, OEBit s,
RCBit r) {
sub(dst, src, value, s, r);
}
void MacroAssembler::SubS64(Register dst, Register src, const Operand& value,
OEBit s, RCBit r) {
if (is_int16(value.immediate()) && s == LeaveOE && r == LeaveRC) {
subi(dst, src, value);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, value);
sub(dst, src, scratch, s, r);
}
}
void MacroAssembler::AddS32(Register dst, Register src, Register value,
RCBit r) {
AddS64(dst, src, value, LeaveOE, r);
extsw(dst, dst, r);
}
void MacroAssembler::AddS32(Register dst, Register src, const Operand& value,
RCBit r) {
AddS64(dst, src, value, LeaveOE, r);
extsw(dst, dst, r);
}
void MacroAssembler::SubS32(Register dst, Register src, Register value,
RCBit r) {
SubS64(dst, src, value, LeaveOE, r);
extsw(dst, dst, r);
}
void MacroAssembler::SubS32(Register dst, Register src, const Operand& value,
RCBit r) {
SubS64(dst, src, value, LeaveOE, r);
extsw(dst, dst, r);
}
void MacroAssembler::MulS64(Register dst, Register src, const Operand& value,
OEBit s, RCBit r) {
if (is_int16(value.immediate()) && s == LeaveOE && r == LeaveRC) {
mulli(dst, src, value);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, value);
mulld(dst, src, scratch, s, r);
}
}
void MacroAssembler::MulS64(Register dst, Register src, Register value, OEBit s,
RCBit r) {
mulld(dst, src, value, s, r);
}
void MacroAssembler::MulS32(Register dst, Register src, const Operand& value,
OEBit s, RCBit r) {
MulS64(dst, src, value, s, r);
extsw(dst, dst, r);
}
void MacroAssembler::MulS32(Register dst, Register src, Register value, OEBit s,
RCBit r) {
MulS64(dst, src, value, s, r);
extsw(dst, dst, r);
}
void MacroAssembler::DivS64(Register dst, Register src, Register value, OEBit s,
RCBit r) {
divd(dst, src, value, s, r);
}
void MacroAssembler::DivU64(Register dst, Register src, Register value, OEBit s,
RCBit r) {
divdu(dst, src, value, s, r);
}
void MacroAssembler::DivS32(Register dst, Register src, Register value, OEBit s,
RCBit r) {
divw(dst, src, value, s, r);
extsw(dst, dst);
}
void MacroAssembler::DivU32(Register dst, Register src, Register value, OEBit s,
RCBit r) {
divwu(dst, src, value, s, r);
ZeroExtWord32(dst, dst);
}
void MacroAssembler::ModS64(Register dst, Register src, Register value) {
modsd(dst, src, value);
}
void MacroAssembler::ModU64(Register dst, Register src, Register value) {
modud(dst, src, value);
}
void MacroAssembler::ModS32(Register dst, Register src, Register value) {
modsw(dst, src, value);
extsw(dst, dst);
}
void MacroAssembler::ModU32(Register dst, Register src, Register value) {
moduw(dst, src, value);
ZeroExtWord32(dst, dst);
}
void MacroAssembler::AndU64(Register dst, Register src, const Operand& value,
RCBit r) {
if (is_uint16(value.immediate()) && r == SetRC) {
andi(dst, src, value);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, value);
and_(dst, src, scratch, r);
}
}
void MacroAssembler::AndU64(Register dst, Register src, Register value,
RCBit r) {
and_(dst, src, value, r);
}
void MacroAssembler::OrU64(Register dst, Register src, const Operand& value,
RCBit r) {
if (is_int16(value.immediate()) && r == LeaveRC) {
ori(dst, src, value);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, value);
orx(dst, src, scratch, r);
}
}
void MacroAssembler::OrU64(Register dst, Register src, Register value,
RCBit r) {
orx(dst, src, value, r);
}
void MacroAssembler::XorU64(Register dst, Register src, const Operand& value,
RCBit r) {
if (is_int16(value.immediate()) && r == LeaveRC) {
xori(dst, src, value);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, value);
xor_(dst, src, scratch, r);
}
}
void MacroAssembler::XorU64(Register dst, Register src, Register value,
RCBit r) {
xor_(dst, src, value, r);
}
void MacroAssembler::AndU32(Register dst, Register src, const Operand& value,
RCBit r) {
AndU64(dst, src, value, r);
extsw(dst, dst, r);
}
void MacroAssembler::AndU32(Register dst, Register src, Register value,
RCBit r) {
AndU64(dst, src, value, r);
extsw(dst, dst, r);
}
void MacroAssembler::OrU32(Register dst, Register src, const Operand& value,
RCBit r) {
OrU64(dst, src, value, r);
extsw(dst, dst, r);
}
void MacroAssembler::OrU32(Register dst, Register src, Register value,
RCBit r) {
OrU64(dst, src, value, r);
extsw(dst, dst, r);
}
void MacroAssembler::XorU32(Register dst, Register src, const Operand& value,
RCBit r) {
XorU64(dst, src, value, r);
extsw(dst, dst, r);
}
void MacroAssembler::XorU32(Register dst, Register src, Register value,
RCBit r) {
XorU64(dst, src, value, r);
extsw(dst, dst, r);
}
void MacroAssembler::ShiftLeftU64(Register dst, Register src,
const Operand& value, RCBit r) {
sldi(dst, src, value, r);
}
void MacroAssembler::ShiftRightU64(Register dst, Register src,
const Operand& value, RCBit r) {
srdi(dst, src, value, r);
}
void MacroAssembler::ShiftRightS64(Register dst, Register src,
const Operand& value, RCBit r) {
sradi(dst, src, value.immediate(), r);
}
void MacroAssembler::ShiftLeftU32(Register dst, Register src,
const Operand& value, RCBit r) {
slwi(dst, src, value, r);
}
void MacroAssembler::ShiftRightU32(Register dst, Register src,
const Operand& value, RCBit r) {
srwi(dst, src, value, r);
}
void MacroAssembler::ShiftRightS32(Register dst, Register src,
const Operand& value, RCBit r) {
srawi(dst, src, value.immediate(), r);
}
void MacroAssembler::ShiftLeftU64(Register dst, Register src, Register value,
RCBit r) {
sld(dst, src, value, r);
}
void MacroAssembler::ShiftRightU64(Register dst, Register src, Register value,
RCBit r) {
srd(dst, src, value, r);
}
void MacroAssembler::ShiftRightS64(Register dst, Register src, Register value,
RCBit r) {
srad(dst, src, value, r);
}
void MacroAssembler::ShiftLeftU32(Register dst, Register src, Register value,
RCBit r) {
slw(dst, src, value, r);
}
void MacroAssembler::ShiftRightU32(Register dst, Register src, Register value,
RCBit r) {
srw(dst, src, value, r);
}
void MacroAssembler::ShiftRightS32(Register dst, Register src, Register value,
RCBit r) {
sraw(dst, src, value, r);
}
void MacroAssembler::CmpS64(Register src1, Register src2, CRegister cr) {
cmp(src1, src2, cr);
}
void MacroAssembler::CmpS64(Register src1, const Operand& src2, CRegister cr) {
intptr_t value = src2.immediate();
if (is_int16(value)) {
cmpi(src1, src2, cr);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, src2);
CmpS64(src1, scratch, cr);
}
}
void MacroAssembler::CmpU64(Register src1, const Operand& src2, CRegister cr) {
intptr_t value = src2.immediate();
if (is_uint16(value)) {
cmpli(src1, src2, cr);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, src2);
CmpU64(src1, scratch, cr);
}
}
void MacroAssembler::CmpU64(Register src1, Register src2, CRegister cr) {
cmpl(src1, src2, cr);
}
void MacroAssembler::CmpS32(Register src1, const Operand& src2, CRegister cr) {
intptr_t value = src2.immediate();
if (is_int16(value)) {
cmpwi(src1, src2, cr);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, src2);
CmpS32(src1, scratch, cr);
}
}
void MacroAssembler::CmpS32(Register src1, Register src2, CRegister cr) {
cmpw(src1, src2, cr);
}
void MacroAssembler::CmpU32(Register src1, const Operand& src2, CRegister cr) {
intptr_t value = src2.immediate();
if (is_uint16(value)) {
cmplwi(src1, src2, cr);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mov(scratch, src2);
CmpU32(src1, scratch, cr);
}
}
void MacroAssembler::CmpU32(Register src1, Register src2, CRegister cr) {
cmplw(src1, src2, cr);
}
void MacroAssembler::AddF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fadd(dst, lhs, rhs, r);
}
void MacroAssembler::SubF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fsub(dst, lhs, rhs, r);
}
void MacroAssembler::MulF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fmul(dst, lhs, rhs, r);
}
void MacroAssembler::DivF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fdiv(dst, lhs, rhs, r);
}
void MacroAssembler::AddF32(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fadd(dst, lhs, rhs, r);
frsp(dst, dst, r);
}
void MacroAssembler::SubF32(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fsub(dst, lhs, rhs, r);
frsp(dst, dst, r);
}
void MacroAssembler::MulF32(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fmul(dst, lhs, rhs, r);
frsp(dst, dst, r);
}
void MacroAssembler::DivF32(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fdiv(dst, lhs, rhs, r);
frsp(dst, dst, r);
}
void MacroAssembler::CopySignF64(DoubleRegister dst, DoubleRegister lhs,
DoubleRegister rhs, RCBit r) {
fcpsgn(dst, rhs, lhs, r);
}
void MacroAssembler::CmpSmiLiteral(Register src1, Tagged<Smi> smi,
CRegister cr) {
#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
CmpS32(src1, Operand(smi), cr);
#else
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadSmiLiteral(scratch, smi);
CmpS64(src1, scratch, cr);
#endif
}
void MacroAssembler::CmplSmiLiteral(Register src1, Tagged<Smi> smi,
CRegister cr) {
#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
CmpU64(src1, Operand(smi), cr);
#else
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadSmiLiteral(scratch, smi);
CmpU64(src1, scratch, cr);
#endif
}
void MacroAssembler::AddSmiLiteral(Register dst, Register src,
Tagged<Smi> smi) {
#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
AddS64(dst, src, Operand(smi.ptr()));
#else
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadSmiLiteral(scratch, smi);
add(dst, src, scratch);
#endif
}
void MacroAssembler::SubSmiLiteral(Register dst, Register src,
Tagged<Smi> smi) {
#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
AddS64(dst, src, Operand(-(static_cast<intptr_t>(smi.ptr()))));
#else
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadSmiLiteral(scratch, smi);
sub(dst, src, scratch);
#endif
}
void MacroAssembler::AndSmiLiteral(Register dst, Register src, Tagged<Smi> smi,
RCBit rc) {
#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
AndU64(dst, src, Operand(smi), rc);
#else
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadSmiLiteral(scratch, smi);
and_(dst, src, scratch, rc);
#endif
}
#define GenerateMemoryOperation(reg, mem, ri_op, rr_op) \
{ \
int64_t offset = mem.offset(); \
\
if (mem.rb() == no_reg) { \
if (!is_int16(offset)) { \
/* cannot use d-form */ \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
ri_op(reg, mem); \
} \
} else { \
if (offset == 0) { \
rr_op(reg, mem); \
} else if (is_int16(offset)) { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
addi(scratch, mem.rb(), Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
add(scratch, scratch, mem.rb()); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} \
} \
}
#define GenerateMemoryOperationRR(reg, mem, op) \
{ \
if (mem.offset() == 0) { \
if (mem.rb() != no_reg) \
op(reg, mem); \
else \
op(reg, MemOperand(r0, mem.ra())); \
} else if (is_int16(mem.offset())) { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
if (mem.rb() != no_reg) \
addi(scratch, mem.rb(), Operand(mem.offset())); \
else \
mov(scratch, Operand(mem.offset())); \
op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(mem.offset())); \
if (mem.rb() != no_reg) add(scratch, scratch, mem.rb()); \
op(reg, MemOperand(mem.ra(), scratch)); \
} \
}
#define GenerateMemoryOperationPrefixed(reg, mem, ri_op, rip_op, rr_op) \
{ \
int64_t offset = mem.offset(); \
\
if (mem.rb() == no_reg) { \
if (is_int16(offset)) { \
ri_op(reg, mem); \
} else if (is_int34(offset) && CpuFeatures::IsSupported(PPC_10_PLUS)) { \
rip_op(reg, mem); \
} else { \
/* cannot use d-form */ \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} \
} else { \
if (offset == 0) { \
rr_op(reg, mem); \
} else if (is_int16(offset)) { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
addi(scratch, mem.rb(), Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
add(scratch, scratch, mem.rb()); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} \
} \
}
#define GenerateMemoryOperationWithAlign(reg, mem, ri_op, rr_op) \
{ \
int64_t offset = mem.offset(); \
int misaligned = (offset & 3); \
\
if (mem.rb() == no_reg) { \
if (!is_int16(offset) || misaligned) { \
/* cannot use d-form */ \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
ri_op(reg, mem); \
} \
} else { \
if (offset == 0) { \
rr_op(reg, mem); \
} else if (is_int16(offset)) { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
addi(scratch, mem.rb(), Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
add(scratch, scratch, mem.rb()); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} \
} \
}
#define GenerateMemoryOperationWithAlignPrefixed(reg, mem, ri_op, rip_op, \
rr_op) \
{ \
int64_t offset = mem.offset(); \
int misaligned = (offset & 3); \
\
if (mem.rb() == no_reg) { \
if (is_int16(offset) && !misaligned) { \
ri_op(reg, mem); \
} else if (is_int34(offset) && CpuFeatures::IsSupported(PPC_10_PLUS)) { \
rip_op(reg, mem); \
} else { \
/* cannot use d-form */ \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} \
} else { \
if (offset == 0) { \
rr_op(reg, mem); \
} else if (is_int16(offset)) { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
addi(scratch, mem.rb(), Operand(offset)); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} else { \
UseScratchRegisterScope temps(this); \
Register scratch = temps.Acquire(); \
mov(scratch, Operand(offset)); \
add(scratch, scratch, mem.rb()); \
rr_op(reg, MemOperand(mem.ra(), scratch)); \
} \
} \
}
#define MEM_OP_WITH_ALIGN_LIST(V) \
V(LoadU64WithUpdate, ldu, ldux) \
V(StoreU64WithUpdate, stdu, stdux)
#define MEM_OP_WITH_ALIGN_FUNCTION(name, ri_op, rr_op) \
void MacroAssembler::name(Register reg, const MemOperand& mem) { \
GenerateMemoryOperationWithAlign(reg, mem, ri_op, rr_op); \
}
MEM_OP_WITH_ALIGN_LIST(MEM_OP_WITH_ALIGN_FUNCTION)
#undef MEM_OP_WITH_ALIGN_LIST
#undef MEM_OP_WITH_ALIGN_FUNCTION
#define MEM_OP_WITH_ALIGN_PREFIXED_LIST(V) \
V(LoadS32, lwa, plwa, lwax) \
V(LoadU64, ld, pld, ldx) \
V(StoreU64, std, pstd, stdx)
#define MEM_OP_WITH_ALIGN_PREFIXED_FUNCTION(name, ri_op, rip_op, rr_op) \
void MacroAssembler::name(Register reg, const MemOperand& mem) { \
GenerateMemoryOperationWithAlignPrefixed(reg, mem, ri_op, rip_op, rr_op); \
}
MEM_OP_WITH_ALIGN_PREFIXED_LIST(MEM_OP_WITH_ALIGN_PREFIXED_FUNCTION)
#undef MEM_OP_WITH_ALIGN_PREFIXED_LIST
#undef MEM_OP_WITH_ALIGN_PREFIXED_FUNCTION
#define MEM_OP_LIST(V) \
V(LoadF64WithUpdate, DoubleRegister, lfdu, lfdux) \
V(LoadF32WithUpdate, DoubleRegister, lfsu, lfsux) \
V(StoreF64WithUpdate, DoubleRegister, stfdu, stfdux) \
V(StoreF32WithUpdate, DoubleRegister, stfsu, stfsux)
#define MEM_OP_FUNCTION(name, result_t, ri_op, rr_op) \
void MacroAssembler::name(result_t reg, const MemOperand& mem) { \
GenerateMemoryOperation(reg, mem, ri_op, rr_op); \
}
MEM_OP_LIST(MEM_OP_FUNCTION)
#undef MEM_OP_LIST
#undef MEM_OP_FUNCTION
#define MEM_OP_PREFIXED_LIST(V) \
V(LoadU32, Register, lwz, plwz, lwzx) \
V(LoadS16, Register, lha, plha, lhax) \
V(LoadU16, Register, lhz, plhz, lhzx) \
V(LoadU8, Register, lbz, plbz, lbzx) \
V(StoreU32, Register, stw, pstw, stwx) \
V(StoreU16, Register, sth, psth, sthx) \
V(StoreU8, Register, stb, pstb, stbx) \
V(LoadF64, DoubleRegister, lfd, plfd, lfdx) \
V(LoadF32, DoubleRegister, lfs, plfs, lfsx) \
V(StoreF64, DoubleRegister, stfd, pstfd, stfdx) \
V(StoreF32, DoubleRegister, stfs, pstfs, stfsx)
#define MEM_OP_PREFIXED_FUNCTION(name, result_t, ri_op, rip_op, rr_op) \
void MacroAssembler::name(result_t reg, const MemOperand& mem) { \
GenerateMemoryOperationPrefixed(reg, mem, ri_op, rip_op, rr_op); \
}
MEM_OP_PREFIXED_LIST(MEM_OP_PREFIXED_FUNCTION)
#undef MEM_OP_PREFIXED_LIST
#undef MEM_OP_PREFIXED_FUNCTION
#define MEM_OP_SIMD_LIST(V) \
V(LoadSimd128, lxvx) \
V(StoreSimd128, stxvx) \
V(LoadSimd128Uint64, lxsdx) \
V(LoadSimd128Uint32, lxsiwzx) \
V(LoadSimd128Uint16, lxsihzx) \
V(LoadSimd128Uint8, lxsibzx) \
V(StoreSimd128Uint64, stxsdx) \
V(StoreSimd128Uint32, stxsiwx) \
V(StoreSimd128Uint16, stxsihx) \
V(StoreSimd128Uint8, stxsibx)
#define MEM_OP_SIMD_FUNCTION(name, rr_op) \
void MacroAssembler::name(Simd128Register reg, const MemOperand& mem) { \
GenerateMemoryOperationRR(reg, mem, rr_op); \
}
MEM_OP_SIMD_LIST(MEM_OP_SIMD_FUNCTION)
#undef MEM_OP_SIMD_LIST
#undef MEM_OP_SIMD_FUNCTION
void MacroAssembler::LoadS8(Register dst, const MemOperand& mem) {
LoadU8(dst, mem);
extsb(dst, dst);
}
#define MEM_LE_OP_LIST(V) \
V(LoadU64, ldbrx) \
V(LoadU32, lwbrx) \
V(LoadU16, lhbrx) \
V(StoreU64, stdbrx) \
V(StoreU32, stwbrx) \
V(StoreU16, sthbrx)
#ifdef V8_TARGET_BIG_ENDIAN
#define MEM_LE_OP_FUNCTION(name, op) \
void MacroAssembler::name##LE(Register reg, const MemOperand& mem) { \
GenerateMemoryOperationRR(reg, mem, op); \
}
#else
#define MEM_LE_OP_FUNCTION(name, op) \
void MacroAssembler::name##LE(Register reg, const MemOperand& mem) { \
name(reg, mem); \
}
#endif
MEM_LE_OP_LIST(MEM_LE_OP_FUNCTION)
#undef MEM_LE_OP_FUNCTION
#undef MEM_LE_OP_LIST
void MacroAssembler::LoadS32LE(Register dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
LoadU32LE(dst, mem);
extsw(dst, dst);
#else
LoadS32(dst, mem);
#endif
}
void MacroAssembler::LoadS16LE(Register dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
LoadU16LE(dst, mem);
extsh(dst, dst);
#else
LoadS16(dst, mem);
#endif
}
void MacroAssembler::LoadF64LE(DoubleRegister dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadU64LE(scratch, mem);
push(scratch);
LoadF64(dst, MemOperand(sp));
pop(scratch);
#else
LoadF64(dst, mem);
#endif
}
void MacroAssembler::LoadF32LE(DoubleRegister dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadU32LE(scratch, mem);
push(scratch);
LoadF32(dst, MemOperand(sp, 4));
pop(scratch);
#else
LoadF32(dst, mem);
#endif
}
void MacroAssembler::StoreF64LE(DoubleRegister dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
StoreF64(dst, mem);
LoadU64(scratch, mem);
StoreU64LE(scratch, mem);
#else
StoreF64(dst, mem);
#endif
}
void MacroAssembler::StoreF32LE(DoubleRegister dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
StoreF32(dst, mem);
LoadU32(scratch, mem);
StoreU32LE(scratch, mem);
#else
StoreF32(dst, mem);
#endif
}
// Simd Support.
#define SIMD_BINOP_LIST(V) \
V(F64x2Add, xvadddp) \
V(F64x2Sub, xvsubdp) \
V(F64x2Mul, xvmuldp) \
V(F64x2Div, xvdivdp) \
V(F64x2Eq, xvcmpeqdp) \
V(F32x4Add, vaddfp) \
V(F32x4Sub, vsubfp) \
V(F32x4Mul, xvmulsp) \
V(F32x4Div, xvdivsp) \
V(F32x4Min, vminfp) \
V(F32x4Max, vmaxfp) \
V(F32x4Eq, xvcmpeqsp) \
V(I64x2Add, vaddudm) \
V(I64x2Sub, vsubudm) \
V(I64x2Eq, vcmpequd) \
V(I64x2GtS, vcmpgtsd) \
V(I32x4Add, vadduwm) \
V(I32x4Sub, vsubuwm) \
V(I32x4Mul, vmuluwm) \
V(I32x4MinS, vminsw) \
V(I32x4MinU, vminuw) \
V(I32x4MaxS, vmaxsw) \
V(I32x4MaxU, vmaxuw) \
V(I32x4Eq, vcmpequw) \
V(I32x4GtS, vcmpgtsw) \
V(I32x4GtU, vcmpgtuw) \
V(I16x8Add, vadduhm) \
V(I16x8Sub, vsubuhm) \
V(I16x8MinS, vminsh) \
V(I16x8MinU, vminuh) \
V(I16x8MaxS, vmaxsh) \
V(I16x8MaxU, vmaxuh) \
V(I16x8Eq, vcmpequh) \
V(I16x8GtS, vcmpgtsh) \
V(I16x8GtU, vcmpgtuh) \
V(I16x8AddSatS, vaddshs) \
V(I16x8SubSatS, vsubshs) \
V(I16x8AddSatU, vadduhs) \
V(I16x8SubSatU, vsubuhs) \
V(I16x8RoundingAverageU, vavguh) \
V(I8x16Add, vaddubm) \
V(I8x16Sub, vsububm) \
V(I8x16MinS, vminsb) \
V(I8x16MinU, vminub) \
V(I8x16MaxS, vmaxsb) \
V(I8x16MaxU, vmaxub) \
V(I8x16Eq, vcmpequb) \
V(I8x16GtS, vcmpgtsb) \
V(I8x16GtU, vcmpgtub) \
V(I8x16AddSatS, vaddsbs) \
V(I8x16SubSatS, vsubsbs) \
V(I8x16AddSatU, vaddubs) \
V(I8x16SubSatU, vsububs) \
V(I8x16RoundingAverageU, vavgub) \
V(S128And, vand) \
V(S128Or, vor) \
V(S128Xor, vxor) \
V(S128AndNot, vandc)
#define EMIT_SIMD_BINOP(name, op) \
void MacroAssembler::name(Simd128Register dst, Simd128Register src1, \
Simd128Register src2) { \
op(dst, src1, src2); \
}
SIMD_BINOP_LIST(EMIT_SIMD_BINOP)
#undef EMIT_SIMD_BINOP
#undef SIMD_BINOP_LIST
#define SIMD_SHIFT_LIST(V) \
V(I64x2Shl, vsld) \
V(I64x2ShrS, vsrad) \
V(I64x2ShrU, vsrd) \
V(I32x4Shl, vslw) \
V(I32x4ShrS, vsraw) \
V(I32x4ShrU, vsrw) \
V(I16x8Shl, vslh) \
V(I16x8ShrS, vsrah) \
V(I16x8ShrU, vsrh) \
V(I8x16Shl, vslb) \
V(I8x16ShrS, vsrab) \
V(I8x16ShrU, vsrb)
#define EMIT_SIMD_SHIFT(name, op) \
void MacroAssembler::name(Simd128Register dst, Simd128Register src1, \
Register src2, Simd128Register scratch) { \
mtvsrd(scratch, src2); \
vspltb(scratch, scratch, Operand(7)); \
op(dst, src1, scratch); \
} \
void MacroAssembler::name(Simd128Register dst, Simd128Register src1, \
const Operand& src2, Register scratch1, \
Simd128Register scratch2) { \
mov(scratch1, src2); \
name(dst, src1, scratch1, scratch2); \
}
SIMD_SHIFT_LIST(EMIT_SIMD_SHIFT)
#undef EMIT_SIMD_SHIFT
#undef SIMD_SHIFT_LIST
#define SIMD_UNOP_LIST(V) \
V(F64x2Abs, xvabsdp) \
V(F64x2Neg, xvnegdp) \
V(F64x2Sqrt, xvsqrtdp) \
V(F64x2Ceil, xvrdpip) \
V(F64x2Floor, xvrdpim) \
V(F64x2Trunc, xvrdpiz) \
V(F32x4Abs, xvabssp) \
V(F32x4Neg, xvnegsp) \
V(F32x4Sqrt, xvsqrtsp) \
V(F32x4Ceil, xvrspip) \
V(F32x4Floor, xvrspim) \
V(F32x4Trunc, xvrspiz) \
V(F32x4SConvertI32x4, xvcvsxwsp) \
V(F32x4UConvertI32x4, xvcvuxwsp) \
V(I64x2Neg, vnegd) \
V(I64x2SConvertI32x4Low, vupklsw) \
V(I64x2SConvertI32x4High, vupkhsw) \
V(I32x4Neg, vnegw) \
V(I32x4SConvertI16x8Low, vupklsh) \
V(I32x4SConvertI16x8High, vupkhsh) \
V(I32x4UConvertF32x4, xvcvspuxws) \
V(I16x8SConvertI8x16Low, vupklsb) \
V(I16x8SConvertI8x16High, vupkhsb) \
V(I8x16Popcnt, vpopcntb)
#define EMIT_SIMD_UNOP(name, op) \
void MacroAssembler::name(Simd128Register dst, Simd128Register src) { \
op(dst, src); \
}
SIMD_UNOP_LIST(EMIT_SIMD_UNOP)
#undef EMIT_SIMD_UNOP
#undef SIMD_UNOP_LIST
#define EXT_MUL(dst_even, dst_odd, mul_even, mul_odd) \
mul_even(dst_even, src1, src2); \
mul_odd(dst_odd, src1, src2);
#define SIMD_EXT_MUL_LIST(V) \
V(I32x4ExtMulLowI16x8S, vmulesh, vmulosh, vmrglw) \
V(I32x4ExtMulHighI16x8S, vmulesh, vmulosh, vmrghw) \
V(I32x4ExtMulLowI16x8U, vmuleuh, vmulouh, vmrglw) \
V(I32x4ExtMulHighI16x8U, vmuleuh, vmulouh, vmrghw) \
V(I16x8ExtMulLowI8x16S, vmulesb, vmulosb, vmrglh) \
V(I16x8ExtMulHighI8x16S, vmulesb, vmulosb, vmrghh) \
V(I16x8ExtMulLowI8x16U, vmuleub, vmuloub, vmrglh) \
V(I16x8ExtMulHighI8x16U, vmuleub, vmuloub, vmrghh)
#define EMIT_SIMD_EXT_MUL(name, mul_even, mul_odd, merge) \
void MacroAssembler::name(Simd128Register dst, Simd128Register src1, \
Simd128Register src2, Simd128Register scratch) { \
EXT_MUL(scratch, dst, mul_even, mul_odd) \
merge(dst, scratch, dst); \
}
SIMD_EXT_MUL_LIST(EMIT_SIMD_EXT_MUL)
#undef EMIT_SIMD_EXT_MUL
#undef SIMD_EXT_MUL_LIST
#define SIMD_ALL_TRUE_LIST(V) \
V(I64x2AllTrue, vcmpgtud) \
V(I32x4AllTrue, vcmpgtuw) \
V(I16x8AllTrue, vcmpgtuh) \
V(I8x16AllTrue, vcmpgtub)
#define EMIT_SIMD_ALL_TRUE(name, op) \
void MacroAssembler::name(Register dst, Simd128Register src, \
Register scratch1, Register scratch2, \
Simd128Register scratch3) { \
constexpr uint8_t fxm = 0x2; /* field mask. */ \
constexpr int bit_number = 24; \
li(scratch1, Operand(0)); \
li(scratch2, Operand(1)); \
/* Check if all lanes > 0, if not then return false.*/ \
vxor(scratch3, scratch3, scratch3); \
mtcrf(scratch1, fxm); /* Clear cr6.*/ \
op(scratch3, src, scratch3, SetRC); \
isel(dst, scratch2, scratch1, bit_number); \
}
SIMD_ALL_TRUE_LIST(EMIT_SIMD_ALL_TRUE)
#undef EMIT_SIMD_ALL_TRUE
#undef SIMD_ALL_TRUE_LIST
#define SIMD_BITMASK_LIST(V) \
V(I64x2BitMask, vextractdm, 0x8080808080800040) \
V(I32x4BitMask, vextractwm, 0x8080808000204060) \
V(I16x8BitMask, vextracthm, 0x10203040506070)
#define EMIT_SIMD_BITMASK(name, op, indicies) \
void MacroAssembler::name(Register dst, Simd128Register src, \
Register scratch1, Simd128Register scratch2) { \
if (CpuFeatures::IsSupported(PPC_10_PLUS)) { \
op(dst, src); \
} else { \
mov(scratch1, Operand(indicies)); /* Select 0 for the high bits. */ \
mtvsrd(scratch2, scratch1); \
vbpermq(scratch2, src, scratch2); \
vextractub(scratch2, scratch2, Operand(6)); \
mfvsrd(dst, scratch2); \
} \
}
SIMD_BITMASK_LIST(EMIT_SIMD_BITMASK)
#undef EMIT_SIMD_BITMASK
#undef SIMD_BITMASK_LIST
#define SIMD_QFM_LIST(V) \
V(F64x2Qfma, xvmaddmdp) \
V(F64x2Qfms, xvnmsubmdp) \
V(F32x4Qfma, xvmaddmsp) \
V(F32x4Qfms, xvnmsubmsp)
#define EMIT_SIMD_QFM(name, op) \
void MacroAssembler::name(Simd128Register dst, Simd128Register src1, \
Simd128Register src2, Simd128Register src3, \
Simd128Register scratch) { \
Simd128Register dest = dst; \
if (dst != src1) { \
vor(scratch, src1, src1); \
dest = scratch; \
} \
op(dest, src2, src3); \
if (dest != dst) { \
vor(dst, dest, dest); \
} \
}
SIMD_QFM_LIST(EMIT_SIMD_QFM)
#undef EMIT_SIMD_QFM
#undef SIMD_QFM_LIST
void MacroAssembler::I64x2ExtMulLowI32x4S(Simd128Register dst,
Simd128Register src1,
Simd128Register src2,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
EXT_MUL(scratch, dst, vmulesw, vmulosw)
vextractd(scratch, scratch, Operand(1 * lane_width_in_bytes));
vinsertd(dst, scratch, Operand(0));
}
void MacroAssembler::I64x2ExtMulHighI32x4S(Simd128Register dst,
Simd128Register src1,
Simd128Register src2,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
EXT_MUL(scratch, dst, vmulesw, vmulosw)
vinsertd(scratch, dst, Operand(1 * lane_width_in_bytes));
vor(dst, scratch, scratch);
}
void MacroAssembler::I64x2ExtMulLowI32x4U(Simd128Register dst,
Simd128Register src1,
Simd128Register src2,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
EXT_MUL(scratch, dst, vmuleuw, vmulouw)
vextractd(scratch, scratch, Operand(1 * lane_width_in_bytes));
vinsertd(dst, scratch, Operand(0));
}
void MacroAssembler::I64x2ExtMulHighI32x4U(Simd128Register dst,
Simd128Register src1,
Simd128Register src2,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
EXT_MUL(scratch, dst, vmuleuw, vmulouw)
vinsertd(scratch, dst, Operand(1 * lane_width_in_bytes));
vor(dst, scratch, scratch);
}
#undef EXT_MUL
void MacroAssembler::LoadSimd128LE(Simd128Register dst, const MemOperand& mem) {
#ifdef V8_TARGET_BIG_ENDIAN
LoadSimd128(dst, mem);
xxbrq(dst, dst);
#else
LoadSimd128(dst, mem);
#endif
}
void MacroAssembler::StoreSimd128LE(Simd128Register src, const MemOperand& mem,
Simd128Register scratch) {
#ifdef V8_TARGET_BIG_ENDIAN
xxbrq(scratch, src);
StoreSimd128(scratch, mem);
#else
StoreSimd128(src, mem);
#endif
}
void MacroAssembler::F64x2Splat(Simd128Register dst, DoubleRegister src,
Register scratch) {
constexpr int lane_width_in_bytes = 8;
MovDoubleToInt64(scratch, src);
mtvsrd(dst, scratch);
vinsertd(dst, dst, Operand(1 * lane_width_in_bytes));
}
void MacroAssembler::F32x4Splat(Simd128Register dst, DoubleRegister src,
DoubleRegister scratch1, Register scratch2) {
MovFloatToInt(scratch2, src, scratch1);
mtvsrd(dst, scratch2);
vspltw(dst, dst, Operand(1));
}
void MacroAssembler::I64x2Splat(Simd128Register dst, Register src) {
constexpr int lane_width_in_bytes = 8;
mtvsrd(dst, src);
vinsertd(dst, dst, Operand(1 * lane_width_in_bytes));
}
void MacroAssembler::I32x4Splat(Simd128Register dst, Register src) {
mtvsrd(dst, src);
vspltw(dst, dst, Operand(1));
}
void MacroAssembler::I16x8Splat(Simd128Register dst, Register src) {
mtvsrd(dst, src);
vsplth(dst, dst, Operand(3));
}
void MacroAssembler::I8x16Splat(Simd128Register dst, Register src) {
mtvsrd(dst, src);
vspltb(dst, dst, Operand(7));
}
void MacroAssembler::F64x2ExtractLane(DoubleRegister dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch1,
Register scratch2) {
constexpr int lane_width_in_bytes = 8;
vextractd(scratch1, src, Operand((1 - imm_lane_idx) * lane_width_in_bytes));
mfvsrd(scratch2, scratch1);
MovInt64ToDouble(dst, scratch2);
}
void MacroAssembler::F32x4ExtractLane(DoubleRegister dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch1,
Register scratch2) {
constexpr int lane_width_in_bytes = 4;
vextractuw(scratch1, src, Operand((3 - imm_lane_idx) * lane_width_in_bytes));
mfvsrd(scratch2, scratch1);
MovIntToFloat(dst, scratch2);
}
void MacroAssembler::I64x2ExtractLane(Register dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
vextractd(scratch, src, Operand((1 - imm_lane_idx) * lane_width_in_bytes));
mfvsrd(dst, scratch);
}
void MacroAssembler::I32x4ExtractLane(Register dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 4;
vextractuw(scratch, src, Operand((3 - imm_lane_idx) * lane_width_in_bytes));
mfvsrd(dst, scratch);
}
void MacroAssembler::I16x8ExtractLaneU(Register dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 2;
vextractuh(scratch, src, Operand((7 - imm_lane_idx) * lane_width_in_bytes));
mfvsrd(dst, scratch);
}
void MacroAssembler::I16x8ExtractLaneS(Register dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch) {
I16x8ExtractLaneU(dst, src, imm_lane_idx, scratch);
extsh(dst, dst);
}
void MacroAssembler::I8x16ExtractLaneU(Register dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch) {
vextractub(scratch, src, Operand(15 - imm_lane_idx));
mfvsrd(dst, scratch);
}
void MacroAssembler::I8x16ExtractLaneS(Register dst, Simd128Register src,
uint8_t imm_lane_idx,
Simd128Register scratch) {
I8x16ExtractLaneU(dst, src, imm_lane_idx, scratch);
extsb(dst, dst);
}
void MacroAssembler::F64x2ReplaceLane(Simd128Register dst, Simd128Register src1,
DoubleRegister src2, uint8_t imm_lane_idx,
Register scratch1,
Simd128Register scratch2) {
constexpr int lane_width_in_bytes = 8;
if (src1 != dst) {
vor(dst, src1, src1);
}
MovDoubleToInt64(scratch1, src2);
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
vinsd(dst, scratch1, Operand((1 - imm_lane_idx) * lane_width_in_bytes));
} else {
mtvsrd(scratch2, scratch1);
vinsertd(dst, scratch2, Operand((1 - imm_lane_idx) * lane_width_in_bytes));
}
}
void MacroAssembler::F32x4ReplaceLane(Simd128Register dst, Simd128Register src1,
DoubleRegister src2, uint8_t imm_lane_idx,
Register scratch1,
DoubleRegister scratch2,
Simd128Register scratch3) {
constexpr int lane_width_in_bytes = 4;
if (src1 != dst) {
vor(dst, src1, src1);
}
MovFloatToInt(scratch1, src2, scratch2);
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
vinsw(dst, scratch1, Operand((3 - imm_lane_idx) * lane_width_in_bytes));
} else {
mtvsrd(scratch3, scratch1);
vinsertw(dst, scratch3, Operand((3 - imm_lane_idx) * lane_width_in_bytes));
}
}
void MacroAssembler::I64x2ReplaceLane(Simd128Register dst, Simd128Register src1,
Register src2, uint8_t imm_lane_idx,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
if (src1 != dst) {
vor(dst, src1, src1);
}
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
vinsd(dst, src2, Operand((1 - imm_lane_idx) * lane_width_in_bytes));
} else {
mtvsrd(scratch, src2);
vinsertd(dst, scratch, Operand((1 - imm_lane_idx) * lane_width_in_bytes));
}
}
void MacroAssembler::I32x4ReplaceLane(Simd128Register dst, Simd128Register src1,
Register src2, uint8_t imm_lane_idx,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 4;
if (src1 != dst) {
vor(dst, src1, src1);
}
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
vinsw(dst, src2, Operand((3 - imm_lane_idx) * lane_width_in_bytes));
} else {
mtvsrd(scratch, src2);
vinsertw(dst, scratch, Operand((3 - imm_lane_idx) * lane_width_in_bytes));
}
}
void MacroAssembler::I16x8ReplaceLane(Simd128Register dst, Simd128Register src1,
Register src2, uint8_t imm_lane_idx,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 2;
if (src1 != dst) {
vor(dst, src1, src1);
}
mtvsrd(scratch, src2);
vinserth(dst, scratch, Operand((7 - imm_lane_idx) * lane_width_in_bytes));
}
void MacroAssembler::I8x16ReplaceLane(Simd128Register dst, Simd128Register src1,
Register src2, uint8_t imm_lane_idx,
Simd128Register scratch) {
if (src1 != dst) {
vor(dst, src1, src1);
}
mtvsrd(scratch, src2);
vinsertb(dst, scratch, Operand(15 - imm_lane_idx));
}
void MacroAssembler::I64x2Mul(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Register scratch1,
Register scratch2, Register scratch3,
Simd128Register scratch4) {
constexpr int lane_width_in_bytes = 8;
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
vmulld(dst, src1, src2);
} else {
DCHECK(scratch1 != r0);
Register scratch_1 = scratch1;
Register scratch_2 = scratch2;
for (int i = 0; i < 2; i++) {
if (i > 0) {
vextractd(scratch4, src1, Operand(1 * lane_width_in_bytes));
vextractd(dst, src2, Operand(1 * lane_width_in_bytes));
src1 = scratch4;
src2 = dst;
}
mfvsrd(scratch_1, src1);
mfvsrd(scratch_2, src2);
mulld(scratch_1, scratch_1, scratch_2);
scratch_1 = scratch2;
scratch_2 = scratch3;
}
mtvsrdd(dst, scratch1, scratch2);
}
}
void MacroAssembler::I16x8Mul(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
vxor(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
vmladduhm(dst, src1, src2, kSimd128RegZero);
}
#define F64X2_MIN_MAX_NAN(result) \
xvcmpeqdp(scratch2, src1, src1); \
vsel(result, src1, result, scratch2); \
xvcmpeqdp(scratch2, src2, src2); \
vsel(dst, src2, result, scratch2); \
/* Use xvmindp to turn any selected SNANs to QNANs. */ \
xvmindp(dst, dst, dst);
void MacroAssembler::F64x2Min(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch1,
Simd128Register scratch2) {
xvmindp(scratch1, src1, src2);
// We need to check if an input is NAN and preserve it.
F64X2_MIN_MAX_NAN(scratch1)
}
void MacroAssembler::F64x2Max(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch1,
Simd128Register scratch2) {
xvmaxdp(scratch1, src1, src2);
// We need to check if an input is NAN and preserve it.
F64X2_MIN_MAX_NAN(scratch1)
}
#undef F64X2_MIN_MAX_NAN
void MacroAssembler::F64x2Lt(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
xvcmpgtdp(dst, src2, src1);
}
void MacroAssembler::F64x2Le(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
xvcmpgedp(dst, src2, src1);
}
void MacroAssembler::F64x2Ne(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
xvcmpeqdp(scratch, src1, src2);
vnor(dst, scratch, scratch);
}
void MacroAssembler::F32x4Lt(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
xvcmpgtsp(dst, src2, src1);
}
void MacroAssembler::F32x4Le(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
xvcmpgesp(dst, src2, src1);
}
void MacroAssembler::F32x4Ne(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
xvcmpeqsp(scratch, src1, src2);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I64x2Ne(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequd(scratch, src1, src2);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I64x2GeS(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpgtsd(scratch, src2, src1);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I32x4Ne(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequw(scratch, src1, src2);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I32x4GeS(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpgtsw(scratch, src2, src1);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I32x4GeU(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequw(scratch, src1, src2);
vcmpgtuw(dst, src1, src2);
vor(dst, dst, scratch);
}
void MacroAssembler::I16x8Ne(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequh(scratch, src1, src2);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I16x8GeS(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpgtsh(scratch, src2, src1);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I16x8GeU(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequh(scratch, src1, src2);
vcmpgtuh(dst, src1, src2);
vor(dst, dst, scratch);
}
void MacroAssembler::I8x16Ne(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequb(scratch, src1, src2);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I8x16GeS(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpgtsb(scratch, src2, src1);
vnor(dst, scratch, scratch);
}
void MacroAssembler::I8x16GeU(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
vcmpequb(scratch, src1, src2);
vcmpgtub(dst, src1, src2);
vor(dst, dst, scratch);
}
void MacroAssembler::I64x2Abs(Simd128Register dst, Simd128Register src,
Simd128Register scratch) {
constexpr int shift_bits = 63;
xxspltib(scratch, Operand(shift_bits));
vsrad(scratch, src, scratch);
vxor(dst, src, scratch);
vsubudm(dst, dst, scratch);
}
void MacroAssembler::I32x4Abs(Simd128Register dst, Simd128Register src,
Simd128Register scratch) {
constexpr int shift_bits = 31;
xxspltib(scratch, Operand(shift_bits));
vsraw(scratch, src, scratch);
vxor(dst, src, scratch);
vsubuwm(dst, dst, scratch);
}
void MacroAssembler::I16x8Abs(Simd128Register dst, Simd128Register src,
Simd128Register scratch) {
constexpr int shift_bits = 15;
xxspltib(scratch, Operand(shift_bits));
vsrah(scratch, src, scratch);
vxor(dst, src, scratch);
vsubuhm(dst, dst, scratch);
}
void MacroAssembler::I16x8Neg(Simd128Register dst, Simd128Register src,
Simd128Register scratch) {
vspltish(scratch, Operand(1));
vnor(dst, src, src);
vadduhm(dst, scratch, dst);
}
void MacroAssembler::I8x16Abs(Simd128Register dst, Simd128Register src,
Simd128Register scratch) {
constexpr int shift_bits = 7;
xxspltib(scratch, Operand(shift_bits));
vsrab(scratch, src, scratch);
vxor(dst, src, scratch);
vsububm(dst, dst, scratch);
}
void MacroAssembler::I8x16Neg(Simd128Register dst, Simd128Register src,
Simd128Register scratch) {
xxspltib(scratch, Operand(1));
vnor(dst, src, src);
vaddubm(dst, scratch, dst);
}
void MacroAssembler::F64x2Pmin(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
xvcmpgtdp(kScratchSimd128Reg, src1, src2);
vsel(dst, src1, src2, kScratchSimd128Reg);
}
void MacroAssembler::F64x2Pmax(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
xvcmpgtdp(kScratchSimd128Reg, src2, src1);
vsel(dst, src1, src2, kScratchSimd128Reg);
}
void MacroAssembler::F32x4Pmin(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
xvcmpgtsp(kScratchSimd128Reg, src1, src2);
vsel(dst, src1, src2, kScratchSimd128Reg);
}
void MacroAssembler::F32x4Pmax(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register scratch) {
xvcmpgtsp(kScratchSimd128Reg, src2, src1);
vsel(dst, src1, src2, kScratchSimd128Reg);
}
void MacroAssembler::I32x4SConvertF32x4(Simd128Register dst,
Simd128Register src,
Simd128Register scratch) {
// NaN to 0
xvcmpeqsp(scratch, src, src);
vand(scratch, src, scratch);
xvcvspsxws(dst, scratch);
}
void MacroAssembler::I16x8SConvertI32x4(Simd128Register dst,
Simd128Register src1,
Simd128Register src2) {
vpkswss(dst, src2, src1);
}
void MacroAssembler::I16x8UConvertI32x4(Simd128Register dst,
Simd128Register src1,
Simd128Register src2) {
vpkswus(dst, src2, src1);
}
void MacroAssembler::I8x16SConvertI16x8(Simd128Register dst,
Simd128Register src1,
Simd128Register src2) {
vpkshss(dst, src2, src1);
}
void MacroAssembler::I8x16UConvertI16x8(Simd128Register dst,
Simd128Register src1,
Simd128Register src2) {
vpkshus(dst, src2, src1);
}
void MacroAssembler::F64x2ConvertLowI32x4S(Simd128Register dst,
Simd128Register src) {
vupklsw(dst, src);
xvcvsxddp(dst, dst);
}
void MacroAssembler::F64x2ConvertLowI32x4U(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
constexpr int lane_width_in_bytes = 8;
vupklsw(dst, src);
// Zero extend.
mov(scratch1, Operand(0xFFFFFFFF));
mtvsrd(scratch2, scratch1);
vinsertd(scratch2, scratch2, Operand(1 * lane_width_in_bytes));
vand(dst, scratch2, dst);
xvcvuxddp(dst, dst);
}
void MacroAssembler::I64x2UConvertI32x4Low(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
constexpr int lane_width_in_bytes = 8;
vupklsw(dst, src);
// Zero extend.
mov(scratch1, Operand(0xFFFFFFFF));
mtvsrd(scratch2, scratch1);
vinsertd(scratch2, scratch2, Operand(1 * lane_width_in_bytes));
vand(dst, scratch2, dst);
}
void MacroAssembler::I64x2UConvertI32x4High(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
constexpr int lane_width_in_bytes = 8;
vupkhsw(dst, src);
// Zero extend.
mov(scratch1, Operand(0xFFFFFFFF));
mtvsrd(scratch2, scratch1);
vinsertd(scratch2, scratch2, Operand(1 * lane_width_in_bytes));
vand(dst, scratch2, dst);
}
void MacroAssembler::I32x4UConvertI16x8Low(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
vupklsh(dst, src);
// Zero extend.
mov(scratch1, Operand(0xFFFF));
mtvsrd(scratch2, scratch1);
vspltw(scratch2, scratch2, Operand(1));
vand(dst, scratch2, dst);
}
void MacroAssembler::I32x4UConvertI16x8High(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
vupkhsh(dst, src);
// Zero extend.
mov(scratch1, Operand(0xFFFF));
mtvsrd(scratch2, scratch1);
vspltw(scratch2, scratch2, Operand(1));
vand(dst, scratch2, dst);
}
void MacroAssembler::I16x8UConvertI8x16Low(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
vupklsb(dst, src);
// Zero extend.
li(scratch1, Operand(0xFF));
mtvsrd(scratch2, scratch1);
vsplth(scratch2, scratch2, Operand(3));
vand(dst, scratch2, dst);
}
void MacroAssembler::I16x8UConvertI8x16High(Simd128Register dst,
Simd128Register src,
Register scratch1,
Simd128Register scratch2) {
vupkhsb(dst, src);
// Zero extend.
li(scratch1, Operand(0xFF));
mtvsrd(scratch2, scratch1);
vsplth(scratch2, scratch2, Operand(3));
vand(dst, scratch2, dst);
}
void MacroAssembler::I8x16BitMask(Register dst, Simd128Register src,
Register scratch1, Register scratch2,
Simd128Register scratch3) {
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
vextractbm(dst, src);
} else {
DCHECK(scratch1 != r0);
mov(scratch1, Operand(0x8101820283038));
mov(scratch2, Operand(0x4048505860687078));
mtvsrdd(scratch3, scratch1, scratch2);
vbpermq(scratch3, src, scratch3);
mfvsrd(dst, scratch3);
}
}
void MacroAssembler::I32x4DotI16x8S(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
vxor(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
vmsumshm(dst, src1, src2, kSimd128RegZero);
}
void MacroAssembler::I32x4DotI8x16AddS(Simd128Register dst,
Simd128Register src1,
Simd128Register src2,
Simd128Register src3) {
vmsummbm(dst, src1, src2, src3);
}
void MacroAssembler::I16x8DotI8x16S(Simd128Register dst, Simd128Register src1,
Simd128Register src2,
Simd128Register scratch) {
vmulesb(scratch, src1, src2);
vmulosb(dst, src1, src2);
vadduhm(dst, scratch, dst);
}
void MacroAssembler::I16x8Q15MulRSatS(Simd128Register dst, Simd128Register src1,
Simd128Register src2) {
vxor(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
vmhraddshs(dst, src1, src2, kSimd128RegZero);
}
void MacroAssembler::I8x16Swizzle(Simd128Register dst, Simd128Register src1,
Simd128Register src2,
Simd128Register scratch) {
// Saturate the indices to 5 bits. Input indices more than 31 should
// return 0.
xxspltib(scratch, Operand(31));
vminub(scratch, src2, scratch);
// Input needs to be reversed.
xxbrq(dst, src1);
vxor(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
vperm(dst, dst, kSimd128RegZero, scratch);
}
void MacroAssembler::I8x16Shuffle(Simd128Register dst, Simd128Register src1,
Simd128Register src2, uint64_t high,
uint64_t low, Register scratch1,
Register scratch2, Simd128Register scratch3) {
DCHECK(scratch2 != r0);
mov(scratch1, Operand(low));
mov(scratch2, Operand(high));
mtvsrdd(scratch3, scratch2, scratch1);
vperm(dst, src1, src2, scratch3);
}
#define EXT_ADD_PAIRWISE(splat, mul_even, mul_odd, add) \
splat(scratch1, Operand(1)); \
mul_even(scratch2, src, scratch1); \
mul_odd(scratch1, src, scratch1); \
add(dst, scratch2, scratch1);
void MacroAssembler::I32x4ExtAddPairwiseI16x8S(Simd128Register dst,
Simd128Register src,
Simd128Register scratch1,
Simd128Register scratch2) {
EXT_ADD_PAIRWISE(vspltish, vmulesh, vmulosh, vadduwm)
}
void MacroAssembler::I32x4ExtAddPairwiseI16x8U(Simd128Register dst,
Simd128Register src,
Simd128Register scratch1,
Simd128Register scratch2) {
EXT_ADD_PAIRWISE(vspltish, vmuleuh, vmulouh, vadduwm)
}
void MacroAssembler::I16x8ExtAddPairwiseI8x16S(Simd128Register dst,
Simd128Register src,
Simd128Register scratch1,
Simd128Register scratch2) {
EXT_ADD_PAIRWISE(xxspltib, vmulesb, vmulosb, vadduhm)
}
void MacroAssembler::I16x8ExtAddPairwiseI8x16U(Simd128Register dst,
Simd128Register src,
Simd128Register scratch1,
Simd128Register scratch2) {
EXT_ADD_PAIRWISE(xxspltib, vmuleub, vmuloub, vadduhm)
}
#undef EXT_ADD_PAIRWISE
void MacroAssembler::F64x2PromoteLowF32x4(Simd128Register dst,
Simd128Register src) {
constexpr int lane_number = 8;
vextractd(dst, src, Operand(lane_number));
vinsertw(dst, dst, Operand(lane_number));
xvcvspdp(dst, dst);
}
void MacroAssembler::F32x4DemoteF64x2Zero(Simd128Register dst,
Simd128Register src,
Simd128Register scratch) {
constexpr int lane_number = 8;
xvcvdpsp(scratch, src);
vextractuw(dst, scratch, Operand(lane_number));
vinsertw(scratch, dst, Operand(4));
vxor(dst, dst, dst);
vinsertd(dst, scratch, Operand(lane_number));
}
void MacroAssembler::I32x4TruncSatF64x2SZero(Simd128Register dst,
Simd128Register src,
Simd128Register scratch) {
constexpr int lane_number = 8;
// NaN to 0.
xvcmpeqdp(scratch, src, src);
vand(scratch, src, scratch);
xvcvdpsxws(scratch, scratch);
vextractuw(dst, scratch, Operand(lane_number));
vinsertw(scratch, dst, Operand(4));
vxor(dst, dst, dst);
vinsertd(dst, scratch, Operand(lane_number));
}
void MacroAssembler::I32x4TruncSatF64x2UZero(Simd128Register dst,
Simd128Register src,
Simd128Register scratch) {
constexpr int lane_number = 8;
xvcvdpuxws(scratch, src);
vextractuw(dst, scratch, Operand(lane_number));
vinsertw(scratch, dst, Operand(4));
vxor(dst, dst, dst);
vinsertd(dst, scratch, Operand(lane_number));
}
#if V8_TARGET_BIG_ENDIAN
#define MAYBE_REVERSE_BYTES(reg, instr) instr(reg, reg);
#else
#define MAYBE_REVERSE_BYTES(reg, instr)
#endif
void MacroAssembler::LoadLane64LE(Simd128Register dst, const MemOperand& mem,
int lane, Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
LoadSimd128Uint64(scratch, mem);
MAYBE_REVERSE_BYTES(scratch, xxbrd)
vinsertd(dst, scratch, Operand((1 - lane) * lane_width_in_bytes));
}
void MacroAssembler::LoadLane32LE(Simd128Register dst, const MemOperand& mem,
int lane, Simd128Register scratch) {
constexpr int lane_width_in_bytes = 4;
LoadSimd128Uint32(scratch, mem);
MAYBE_REVERSE_BYTES(scratch, xxbrw)
vinsertw(dst, scratch, Operand((3 - lane) * lane_width_in_bytes));
}
void MacroAssembler::LoadLane16LE(Simd128Register dst, const MemOperand& mem,
int lane, Simd128Register scratch) {
constexpr int lane_width_in_bytes = 2;
LoadSimd128Uint16(scratch, mem);
MAYBE_REVERSE_BYTES(scratch, xxbrh)
vinserth(dst, scratch, Operand((7 - lane) * lane_width_in_bytes));
}
void MacroAssembler::LoadLane8LE(Simd128Register dst, const MemOperand& mem,
int lane, Simd128Register scratch) {
LoadSimd128Uint8(scratch, mem);
vinsertb(dst, scratch, Operand((15 - lane)));
}
void MacroAssembler::StoreLane64LE(Simd128Register src, const MemOperand& mem,
int lane, Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
vextractd(scratch, src, Operand((1 - lane) * lane_width_in_bytes));
MAYBE_REVERSE_BYTES(scratch, xxbrd)
StoreSimd128Uint64(scratch, mem);
}
void MacroAssembler::StoreLane32LE(Simd128Register src, const MemOperand& mem,
int lane, Simd128Register scratch) {
constexpr int lane_width_in_bytes = 4;
vextractuw(scratch, src, Operand((3 - lane) * lane_width_in_bytes));
MAYBE_REVERSE_BYTES(scratch, xxbrw)
StoreSimd128Uint32(scratch, mem);
}
void MacroAssembler::StoreLane16LE(Simd128Register src, const MemOperand& mem,
int lane, Simd128Register scratch) {
constexpr int lane_width_in_bytes = 2;
vextractuh(scratch, src, Operand((7 - lane) * lane_width_in_bytes));
MAYBE_REVERSE_BYTES(scratch, xxbrh)
StoreSimd128Uint16(scratch, mem);
}
void MacroAssembler::StoreLane8LE(Simd128Register src, const MemOperand& mem,
int lane, Simd128Register scratch) {
vextractub(scratch, src, Operand(15 - lane));
StoreSimd128Uint8(scratch, mem);
}
void MacroAssembler::LoadAndSplat64x2LE(Simd128Register dst,
const MemOperand& mem) {
constexpr int lane_width_in_bytes = 8;
LoadSimd128Uint64(dst, mem);
MAYBE_REVERSE_BYTES(dst, xxbrd)
vinsertd(dst, dst, Operand(1 * lane_width_in_bytes));
}
void MacroAssembler::LoadAndSplat32x4LE(Simd128Register dst,
const MemOperand& mem) {
LoadSimd128Uint32(dst, mem);
MAYBE_REVERSE_BYTES(dst, xxbrw)
vspltw(dst, dst, Operand(1));
}
void MacroAssembler::LoadAndSplat16x8LE(Simd128Register dst,
const MemOperand& mem) {
LoadSimd128Uint16(dst, mem);
MAYBE_REVERSE_BYTES(dst, xxbrh)
vsplth(dst, dst, Operand(3));
}
void MacroAssembler::LoadAndSplat8x16LE(Simd128Register dst,
const MemOperand& mem) {
LoadSimd128Uint8(dst, mem);
vspltb(dst, dst, Operand(7));
}
void MacroAssembler::LoadAndExtend32x2SLE(Simd128Register dst,
const MemOperand& mem) {
LoadSimd128Uint64(dst, mem);
MAYBE_REVERSE_BYTES(dst, xxbrd)
vupkhsw(dst, dst);
}
void MacroAssembler::LoadAndExtend32x2ULE(Simd128Register dst,
const MemOperand& mem,
Register scratch1,
Simd128Register scratch2) {
constexpr int lane_width_in_bytes = 8;
LoadAndExtend32x2SLE(dst, mem);
// Zero extend.
mov(scratch1, Operand(0xFFFFFFFF));
mtvsrd(scratch2, scratch1);
vinsertd(scratch2, scratch2, Operand(1 * lane_width_in_bytes));
vand(dst, scratch2, dst);
}
void MacroAssembler::LoadAndExtend16x4SLE(Simd128Register dst,
const MemOperand& mem) {
LoadSimd128Uint64(dst, mem);
MAYBE_REVERSE_BYTES(dst, xxbrd)
vupkhsh(dst, dst);
}
void MacroAssembler::LoadAndExtend16x4ULE(Simd128Register dst,
const MemOperand& mem,
Register scratch1,
Simd128Register scratch2) {
LoadAndExtend16x4SLE(dst, mem);
// Zero extend.
mov(scratch1, Operand(0xFFFF));
mtvsrd(scratch2, scratch1);
vspltw(scratch2, scratch2, Operand(1));
vand(dst, scratch2, dst);
}
void MacroAssembler::LoadAndExtend8x8SLE(Simd128Register dst,
const MemOperand& mem) {
LoadSimd128Uint64(dst, mem);
MAYBE_REVERSE_BYTES(dst, xxbrd)
vupkhsb(dst, dst);
}
void MacroAssembler::LoadAndExtend8x8ULE(Simd128Register dst,
const MemOperand& mem,
Register scratch1,
Simd128Register scratch2) {
LoadAndExtend8x8SLE(dst, mem);
// Zero extend.
li(scratch1, Operand(0xFF));
mtvsrd(scratch2, scratch1);
vsplth(scratch2, scratch2, Operand(3));
vand(dst, scratch2, dst);
}
void MacroAssembler::LoadV64ZeroLE(Simd128Register dst, const MemOperand& mem,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 8;
LoadSimd128Uint64(scratch, mem);
MAYBE_REVERSE_BYTES(scratch, xxbrd)
vxor(dst, dst, dst);
vinsertd(dst, scratch, Operand(1 * lane_width_in_bytes));
}
void MacroAssembler::LoadV32ZeroLE(Simd128Register dst, const MemOperand& mem,
Simd128Register scratch) {
constexpr int lane_width_in_bytes = 4;
LoadSimd128Uint32(scratch, mem);
MAYBE_REVERSE_BYTES(scratch, xxbrw)
vxor(dst, dst, dst);
vinsertw(dst, scratch, Operand(3 * lane_width_in_bytes));
}
#undef MAYBE_REVERSE_BYTES
void MacroAssembler::V128AnyTrue(Register dst, Simd128Register src,
Register scratch1, Register scratch2,
Simd128Register scratch3) {
constexpr uint8_t fxm = 0x2; // field mask.
constexpr int bit_number = 24;
li(scratch1, Operand(0));
li(scratch2, Operand(1));
// Check if both lanes are 0, if so then return false.
vxor(scratch3, scratch3, scratch3);
mtcrf(scratch1, fxm); // Clear cr6.
vcmpequd(scratch3, src, scratch3, SetRC);
isel(dst, scratch1, scratch2, bit_number);
}
void MacroAssembler::S128Not(Simd128Register dst, Simd128Register src) {
vnor(dst, src, src);
}
void MacroAssembler::S128Const(Simd128Register dst, uint64_t high, uint64_t low,
Register scratch1, Register scratch2) {
DCHECK(scratch2 != r0);
mov(scratch1, Operand(low));
mov(scratch2, Operand(high));
mtvsrdd(dst, scratch2, scratch1);
}
void MacroAssembler::S128Select(Simd128Register dst, Simd128Register src1,
Simd128Register src2, Simd128Register mask) {
vsel(dst, src2, src1, mask);
}
Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2, Register reg3,
Register reg4, Register reg5,
Register reg6) {
RegList regs = {reg1, reg2, reg3, reg4, reg5, reg6};
const RegisterConfiguration* config = RegisterConfiguration::Default();
for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
int code = config->GetAllocatableGeneralCode(i);
Register candidate = Register::from_code(code);
if (regs.has(candidate)) continue;
return candidate;
}
UNREACHABLE();
}
void MacroAssembler::PreCheckSkippedWriteBarrier(Register object,
Register value,
Register scratch, Label* ok) {
ASM_CODE_COMMENT(this);
DCHECK(!AreAliased(object, scratch));
DCHECK(!AreAliased(value, scratch));
// The most common case: Static write barrier elimination is allowed on the
// last young allocation.
{
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
DCHECK(!AreAliased(scratch, scratch1));
SubS64(scratch, object, Operand(kHeapObjectTag));
LoadU64(scratch1, MemOperand(kRootRegister,
IsolateData::last_young_allocation_offset()));
CmpU64(scratch, scratch1);
b(to_condition(Condition::kEqual), ok);
}
#if CONTIGUOUS_COMPRESSED_READ_ONLY_SPACE_BOOL
JumpIfUnsignedLessThan(value, kContiguousReadOnlyReservationSize, ok);
#else // !CONTIGUOUS_COMPRESSED_READ_ONLY_SPACE_BOOL
// Write barier can also be removed if value is in read-only space.
CheckPageFlag(value, scratch, MemoryChunk::kIsInReadOnlyHeapMask, ne, ok);
#endif // !CONTIGUOUS_COMPRESSED_READ_ONLY_SPACE_BOOL
Label not_ok;
// Handle allocation folding, allow WB removal if:
// LAB start <= last_young_allocation_ < (object address+1) < LAB top
// Note that object has tag bit set, so object == object address+1.
{
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
DCHECK(!AreAliased(scratch, scratch1));
// Check LAB start <= last_young_allocation_.
LoadU64(scratch,
MemOperand(kRootRegister,
IsolateData::new_allocation_info_start_offset()));
LoadU64(scratch1, MemOperand(kRootRegister,
IsolateData::last_young_allocation_offset()));
CmpU64(scratch, scratch1);
b(to_condition(Condition::kUnsignedGreaterThan), ¬_ok);
// Check last_young_allocation_ < (object address+1).
CmpU64(scratch1, object);
b(to_condition(Condition::kUnsignedGreaterThanEqual), ¬_ok);
// Check (object address+1) < LAB top.
LoadU64(scratch, MemOperand(kRootRegister,
IsolateData::new_allocation_info_top_offset()));
CmpU64(object, scratch);
b(to_condition(Condition::kUnsignedLessThan), ok);
}
// Slow path: Potentially check more cases in C++.
bind(¬_ok);
}
void MacroAssembler::SwapP(Register src, Register dst) {
if (src == dst) return;
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mr(scratch, src);
mr(src, dst);
mr(dst, scratch);
}
void MacroAssembler::SwapP(Register src, MemOperand dst) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
mr(scratch, src);
LoadU64(src, dst);
StoreU64(scratch, dst);
}
void MacroAssembler::SwapP(MemOperand src, MemOperand dst) {
UseScratchRegisterScope temps(this);
Register scratch_0 = temps.Acquire();
Register scratch_1 = temps.Acquire();
if (is_int16(src.offset()) || is_int16(dst.offset())) {
if (!is_int16(src.offset())) {
// swap operand
MemOperand temp = src;
src = dst;
dst = temp;
}
LoadU64(scratch_1, dst);
LoadU64(scratch_0, src);
StoreU64(scratch_1, src);
StoreU64(scratch_0, dst);
} else {
LoadU64(scratch_1, dst);
push(scratch_1);
LoadU64(scratch_0, src);
StoreU64(scratch_0, dst);
pop(scratch_1);
StoreU64(scratch_1, src);
}
}
void MacroAssembler::SwapFloat32(DoubleRegister src, DoubleRegister dst,
DoubleRegister scratch) {
if (src == dst) return;
DCHECK(!AreAliased(src, dst, scratch));
fmr(scratch, src);
fmr(src, dst);
fmr(dst, scratch);
}
void MacroAssembler::SwapFloat32(DoubleRegister src, MemOperand dst,
DoubleRegister scratch) {
DCHECK(!AreAliased(src, scratch));
fmr(scratch, src);
LoadF32(src, dst);
StoreF32(scratch, dst);
}
void MacroAssembler::SwapFloat32(MemOperand src, MemOperand dst,
DoubleRegister scratch_0,
DoubleRegister scratch_1) {
DCHECK(!AreAliased(scratch_0, scratch_1));
LoadF32(scratch_0, src);
LoadF32(scratch_1, dst);
StoreF32(scratch_0, dst);
StoreF32(scratch_1, src);
}
void MacroAssembler::SwapDouble(DoubleRegister src, DoubleRegister dst,
DoubleRegister scratch) {
if (src == dst) return;
DCHECK(!AreAliased(src, dst, scratch));
fmr(scratch, src);
fmr(src, dst);
fmr(dst, scratch);
}
void MacroAssembler::SwapDouble(DoubleRegister src, MemOperand dst,
DoubleRegister scratch) {
DCHECK(!AreAliased(src, scratch));
fmr(scratch, src);
LoadF64(src, dst);
StoreF64(scratch, dst);
}
void MacroAssembler::SwapDouble(MemOperand src, MemOperand dst,
DoubleRegister scratch_0,
DoubleRegister scratch_1) {
DCHECK(!AreAliased(scratch_0, scratch_1));
LoadF64(scratch_0, src);
LoadF64(scratch_1, dst);
StoreF64(scratch_0, dst);
StoreF64(scratch_1, src);
}
void MacroAssembler::SwapSimd128(Simd128Register src, Simd128Register dst,
Simd128Register scratch) {
if (src == dst) return;
vor(scratch, src, src);
vor(src, dst, dst);
vor(dst, scratch, scratch);
}
void MacroAssembler::SwapSimd128(Simd128Register src, MemOperand dst,
Simd128Register scratch) {
DCHECK(src != scratch);
LoadSimd128(scratch, dst);
StoreSimd128(src, dst);
vor(src, scratch, scratch);
}
void MacroAssembler::SwapSimd128(MemOperand src, MemOperand dst,
Simd128Register scratch1,
Simd128Register scratch2) {
LoadSimd128(scratch1, src);
LoadSimd128(scratch2, dst);
StoreSimd128(scratch1, dst);
StoreSimd128(scratch2, src);
}
void MacroAssembler::ByteReverseU16(Register dst, Register val) {
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
brh(dst, val);
ZeroExtHalfWord(dst, dst);
return;
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
rlwinm(scratch, val, 8, 16, 23);
rlwinm(dst, val, 24, 24, 31);
orx(dst, scratch, dst);
ZeroExtHalfWord(dst, dst);
}
void MacroAssembler::ByteReverseU32(Register dst, Register val) {
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
brw(dst, val);
ZeroExtWord32(dst, dst);
return;
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
rotlwi(scratch, val, 8);
rlwimi(scratch, val, 24, 0, 7);
rlwimi(scratch, val, 24, 16, 23);
ZeroExtWord32(dst, scratch);
}
void MacroAssembler::ByteReverseU64(Register dst, Register val) {
if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
brd(dst, val);
return;
}
subi(sp, sp, Operand(kSystemPointerSize));
std(val, MemOperand(sp));
ldbrx(dst, MemOperand(r0, sp));
addi(sp, sp, Operand(kSystemPointerSize));
}
void MacroAssembler::JumpIfEqual(Register x, int32_t y, Label* dest) {
CmpS32(x, Operand(y));
beq(dest);
}
void MacroAssembler::JumpIfLessThan(Register x, int32_t y, Label* dest) {
CmpS32(x, Operand(y));
blt(dest);
}
void MacroAssembler::JumpIfUnsignedLessThan(Register x, int32_t y,
Label* dest) {
CmpU32(x, Operand(y));
blt(dest);
}
void MacroAssembler::LoadEntryFromBuiltinIndex(Register builtin_index,
Register target) {
static_assert(kSystemPointerSize == 8);
static_assert(kSmiTagSize == 1);
static_assert(kSmiTag == 0);
// The builtin_index register contains the builtin index as a Smi.
if (SmiValuesAre32Bits()) {
ShiftRightS64(target, builtin_index,
Operand(kSmiShift - kSystemPointerSizeLog2));
} else {
DCHECK(SmiValuesAre31Bits());
ShiftLeftU64(target, builtin_index,
Operand(kSystemPointerSizeLog2 - kSmiShift));
}
AddS64(target, target, Operand(IsolateData::builtin_entry_table_offset()));
LoadU64(target, MemOperand(kRootRegister, target));
}
void MacroAssembler::CallBuiltinByIndex(Register builtin_index,
Register target) {
LoadEntryFromBuiltinIndex(builtin_index, target);
Call(target);
}
void MacroAssembler::LoadEntryFromBuiltin(Builtin builtin,
Register destination) {
ASM_CODE_COMMENT(this);
LoadU64(destination, EntryFromBuiltinAsOperand(builtin));
}
MemOperand MacroAssembler::EntryFromBuiltinAsOperand(Builtin builtin) {
ASM_CODE_COMMENT(this);
DCHECK(root_array_available());
return MemOperand(kRootRegister,
IsolateData::BuiltinEntrySlotOffset(builtin));
}
void MacroAssembler::LoadEntrypointFromJSDispatchTable(
Register destination, Register dispatch_handle) {
ASM_CODE_COMMENT(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Register index = destination;
CHECK(root_array_available());
LoadU64(scratch,
ExternalReferenceAsOperand(IsolateFieldId::kJSDispatchTable));
ShiftRightU64(index, dispatch_handle, Operand(kJSDispatchHandleShift));
ShiftLeftU64(index, index, Operand(kJSDispatchTableEntrySizeLog2));
AddS64(scratch, scratch, index);
LoadU64(destination, MemOperand(scratch, JSDispatchEntry::kEntrypointOffset));
}
void MacroAssembler::LoadCodeInstructionStart(Register destination,
Register code_object,
CodeEntrypointTag tag) {
ASM_CODE_COMMENT(this);
LoadU64(destination,
FieldMemOperand(code_object, Code::kInstructionStartOffset));
}
void MacroAssembler::CallCodeObject(Register code_object) {
ASM_CODE_COMMENT(this);
LoadCodeInstructionStart(code_object, code_object);
Call(code_object);
}
void MacroAssembler::JumpCodeObject(Register code_object, JumpMode jump_mode) {
ASM_CODE_COMMENT(this);
DCHECK_EQ(JumpMode::kJump, jump_mode);
LoadCodeInstructionStart(code_object, code_object);
Jump(code_object);
}
void MacroAssembler::CallJSFunction(Register function_object,
uint16_t argument_count) {
Register code = kJavaScriptCallCodeStartRegister;
UseScratchRegisterScope temps(this);
Register dispatch_handle = temps.Acquire();
LoadU32(
dispatch_handle,
FieldMemOperand(function_object, offsetof(JSFunction, dispatch_handle_)));
LoadEntrypointFromJSDispatchTable(code, dispatch_handle);
Call(code);
}
void MacroAssembler::CallJSDispatchEntry(JSDispatchHandle dispatch_handle,
uint16_t argument_count) {
Register code = kJavaScriptCallCodeStartRegister;
UseScratchRegisterScope temps(this);
Register dispatch_handle_reg = temps.Acquire();
mov(dispatch_handle_reg,
Operand(dispatch_handle.value(), RelocInfo::JS_DISPATCH_HANDLE));
// WARNING: This entrypoint load is only safe because we are storing a
// RelocInfo for the dispatch handle in the movl above (thus keeping the
// dispatch entry alive) _and_ because the entrypoints are not compactable
// (thus meaning that the calculation in the entrypoint load is not
// invalidated by a compaction).
// TODO(leszeks): Make this less of a footgun.
static_assert(!JSDispatchTable::kSupportsCompaction);
LoadEntrypointFromJSDispatchTable(code, dispatch_handle_reg);
CHECK_EQ(argument_count,
isolate()->js_dispatch_table().GetParameterCount(dispatch_handle));
Call(code);
}
void MacroAssembler::JumpJSFunction(Register function_object,
JumpMode jump_mode) {
Register code = kJavaScriptCallCodeStartRegister;
UseScratchRegisterScope temps(this);
Register dispatch_handle = temps.Acquire();
LoadU32(
dispatch_handle,
FieldMemOperand(function_object, offsetof(JSFunction, dispatch_handle_)));
LoadEntrypointFromJSDispatchTable(code, dispatch_handle);
Jump(code);
}
#ifdef V8_ENABLE_WEBASSEMBLY
void MacroAssembler::ResolveWasmCodePointer(Register target) {
ASM_CODE_COMMENT(this);
static_assert(!V8_ENABLE_SANDBOX_BOOL);
ExternalReference global_jump_table =
ExternalReference::wasm_code_pointer_table();
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Move(scratch, global_jump_table);
static_assert(sizeof(wasm::WasmCodePointerTableEntry) == kSystemPointerSize);
ShiftLeftU32(target, target, Operand(kSystemPointerSizeLog2));
LoadU64(target, MemOperand(scratch, target));
}
void MacroAssembler::CallWasmCodePointer(Register target,
CallJumpMode call_jump_mode) {
ResolveWasmCodePointer(target);
if (call_jump_mode == CallJumpMode::kTailCall) {
Jump(target);
} else {
Call(target);
}
}
void MacroAssembler::LoadWasmCodePointer(Register dst, MemOperand src) {
static_assert(sizeof(WasmCodePointer) == 4);
LoadU32(dst, src);
}
#endif
void MacroAssembler::StoreReturnAddressAndCall(Register target) {
// This generates the final instruction sequence for calls to C functions
// once an exit frame has been constructed.
//
// Note that this assumes the caller code (i.e. the InstructionStream object
// currently being generated) is immovable or that the callee function cannot
// trigger GC, since the callee function will return to it.
Label return_label;
Register dest = target;
Register scratch = r7;
DCHECK(!AreAliased(r0, scratch, target));
GetLabelAddress(scratch, &return_label);
if (ABI_USES_FUNCTION_DESCRIPTORS) {
// AIX/PPC64BE Linux uses a function descriptor. When calling C code be
// aware of this descriptor and pick up values from it
LoadU64(ToRegister(ABI_TOC_REGISTER),
MemOperand(target, kSystemPointerSize));
LoadU64(ip, MemOperand(target, 0));
dest = ip;
} else if (ABI_CALL_VIA_IP && dest != ip) {
Move(ip, target);
dest = ip;
}
StoreU64(scratch,
MemOperand(sp, kStackFrameExtraParamSlot * kSystemPointerSize));
Call(dest);
bind(&return_label);
}
void MacroAssembler::AssertNotDeoptimized(Register scratch) {
DCHECK(!AreAliased(scratch, r0));
int offset = InstructionStream::kCodeOffset - InstructionStream::kHeaderSize;
LoadTaggedField(scratch,
MemOperand(kJavaScriptCallCodeStartRegister, offset));
TestCodeIsMarkedForDeoptimization(scratch);
Assert(to_condition(kZero), AbortReason::kInvalidDeoptimizedCode);
}
void MacroAssembler::CallForDeoptimization(Builtin target, int, Label* exit,
DeoptimizeKind kind, Label* ret,
Label*) {
BlockTrampolinePoolScope block_trampoline_pool(this);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
CHECK_LE(target, Builtins::kLastTier0);
LoadU64(scratch, MemOperand(kRootRegister,
IsolateData::BuiltinEntrySlotOffset(target)));
Call(scratch);
DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
(kind == DeoptimizeKind::kLazy) ? Deoptimizer::kLazyDeoptExitSize
: Deoptimizer::kEagerDeoptExitSize);
}
void MacroAssembler::ZeroExtByte(Register dst, Register src) {
clrldi(dst, src, Operand(56));
}
void MacroAssembler::ZeroExtHalfWord(Register dst, Register src) {
clrldi(dst, src, Operand(48));
}
void MacroAssembler::ZeroExtWord32(Register dst, Register src) {
clrldi(dst, src, Operand(32));
}
void MacroAssembler::Trap() { stop(); }
void MacroAssembler::DebugBreak() { stop(); }
void MacroAssembler::Popcnt32(Register dst, Register src) { popcntw(dst, src); }
void MacroAssembler::Popcnt64(Register dst, Register src) { popcntd(dst, src); }
void MacroAssembler::CountLeadingZerosU32(Register dst, Register src, RCBit r) {
cntlzw(dst, src, r);
}
void MacroAssembler::CountLeadingZerosU64(Register dst, Register src, RCBit r) {
cntlzd(dst, src, r);
}
void MacroAssembler::CountTrailingZerosU32(Register dst, Register src,
RCBit r) {
cnttzw(dst, src, r);
}
void MacroAssembler::CountTrailingZerosU64(Register dst, Register src,
RCBit r) {
cnttzd(dst, src, r);
}
void MacroAssembler::ClearByteU64(Register dst, int byte_idx) {
CHECK(0 <= byte_idx && byte_idx <= 7);
int shift = byte_idx*8;
rldicl(dst, dst, shift, 8);
rldicl(dst, dst, 64-shift, 0);
}
void MacroAssembler::ReverseBitsU64(Register dst, Register src) {
ByteReverseU64(dst, src);
for (int i = 0; i < 8; i++) {
ReverseBitsInSingleByteU64(dst, dst, i);
}
}
void MacroAssembler::ReverseBitsU32(Register dst, Register src) {
ByteReverseU32(dst, src);
for (int i = 4; i < 8; i++) {
ReverseBitsInSingleByteU64(dst, dst, i);
}
}
// byte_idx=7 refers to least significant byte
void MacroAssembler::ReverseBitsInSingleByteU64(Register dst, Register src,
int byte_idx) {
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
Register scratch2 = temps.Acquire();
CHECK(0 <= byte_idx && byte_idx <= 7);
int j = byte_idx;
// zero all bits of scratch1
li(scratch2, Operand(0));
for (int i = 0; i <= 7; i++) {
// zero all bits of scratch1
li(scratch1, Operand(0));
// move bit (j+1)*8-i-1 of src to bit j*8+i of scratch1, erase bits
// (j*8+i+1):end of scratch1
int shift = 7 - (2*i);
if (shift < 0) shift += 64;
rldicr(scratch1, src, shift, j*8+i);
// erase bits start:(j*8-1+i) of scratch1 (inclusive)
rldicl(scratch1, scratch1, 0, j*8+i);
// scratch2 = scratch2|scratch1
orx(scratch2, scratch2, scratch1);
}
// clear jth byte of dst and insert jth byte of scratch2
ClearByteU64(dst, j);
orx(dst, dst, scratch2);
}
void MacroAssembler::JumpIfCodeIsMarkedForDeoptimization(
Register code, Label* if_marked_for_deoptimization) {
TestCodeIsMarkedForDeoptimization(code);
bne(if_marked_for_deoptimization);
}
void MacroAssembler::JumpIfCodeIsTurbofanned(Register code,
Label* if_turbofanned) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
LoadU32(scratch, FieldMemOperand(code, Code::kFlagsOffset));
TestBit(scratch, Code::kIsTurbofannedBit);
bne(if_turbofanned);
}
void MacroAssembler::TryLoadOptimizedOsrCode(Register scratch_and_result,
CodeKind min_opt_level,
Register feedback_vector,
FeedbackSlot slot,
Label* on_result,
Label::Distance) {
Label fallthrough, clear_slot;
LoadTaggedField(
scratch_and_result,
FieldMemOperand(feedback_vector,
FeedbackVector::OffsetOfElementAt(slot.ToInt())));
LoadWeakValue(scratch_and_result, scratch_and_result, &fallthrough);
// Is it marked_for_deoptimization? If yes, clear the slot.
{
// The entry references a CodeWrapper object. Unwrap it now.
LoadTaggedField(
scratch_and_result,
FieldMemOperand(scratch_and_result, offsetof(CodeWrapper, code_)));
JumpIfCodeIsMarkedForDeoptimization(scratch_and_result, &clear_slot);
if (min_opt_level == CodeKind::TURBOFAN_JS) {
JumpIfCodeIsTurbofanned(scratch_and_result, on_result);
b(&fallthrough);
} else {
b(on_result);
}
}
bind(&clear_slot);
mov(scratch_and_result, ClearedValue());
StoreTaggedField(
scratch_and_result,
FieldMemOperand(feedback_vector,
FeedbackVector::OffsetOfElementAt(slot.ToInt())));
bind(&fallthrough);
mov(scratch_and_result, Operand::Zero());
}
// Calls an API function. Allocates HandleScope, extracts returned value
// from handle and propagates exceptions. Clobbers C argument registers
// and C caller-saved registers. Restores context. On return removes
// (*argc_operand + slots_to_drop_on_return) * kSystemPointerSize
// (GCed, includes the call JS arguments space and the additional space
// allocated for the fast call).
void CallApiFunctionAndReturn(MacroAssembler* masm, bool with_profiling,
Register function_address,
ExternalReference thunk_ref, Register thunk_arg,
int slots_to_drop_on_return,
MemOperand* argc_operand,
MemOperand return_value_operand,
bool handle_interceptor_result) {
using ER = ExternalReference;
Isolate* isolate = masm->isolate();
MemOperand next_mem_op = __ ExternalReferenceAsOperand(
ER::handle_scope_next_address(isolate), no_reg);
MemOperand limit_mem_op = __ ExternalReferenceAsOperand(
ER::handle_scope_limit_address(isolate), no_reg);
MemOperand level_mem_op = __ ExternalReferenceAsOperand(
ER::handle_scope_level_address(isolate), no_reg);
// Additional parameter is the address of the actual callback.
Register return_value = r3;
UseScratchRegisterScope temps(masm);
Register scratch = temps.Acquire();
// Allocate HandleScope in callee-saved registers.
// We will need to restore the HandleScope after the call to the API function,
// by allocating it in callee-saved registers it'll be preserved by C code.
Register prev_next_address_reg = r14;
Register prev_limit_reg = r15;
Register prev_level_reg = r16;
// C arguments (kCArgRegs[0/1/2]) are expected to be initialized outside, so
// this function must not corrupt them (return_value overlaps with
// kCArgRegs[0] but that's ok because we start using it only after the C
// call).
DCHECK(!AreAliased(kCArgRegs[0], kCArgRegs[1], kCArgRegs[2], // C args
scratch, prev_next_address_reg, prev_limit_reg));
// function_address and thunk_arg might overlap but this function must not
// corrupt them until the call is made (i.e. overlap with return_value is
// fine).
DCHECK(!AreAliased(function_address, // incoming parameters
scratch, prev_next_address_reg, prev_limit_reg));
DCHECK(!AreAliased(thunk_arg, // incoming parameters
scratch, prev_next_address_reg, prev_limit_reg));
{
ASM_CODE_COMMENT_STRING(masm,
"Allocate HandleScope in callee-save registers.");
__ LoadU64(prev_next_address_reg, next_mem_op);
__ LoadU64(prev_limit_reg, limit_mem_op);
__ lwz(prev_level_reg, level_mem_op);
__ addi(scratch, prev_level_reg, Operand(1));
__ stw(scratch, level_mem_op);
}
Label profiler_or_side_effects_check_enabled, done_api_call,
done_reading_result;
if (with_profiling) {
__ RecordComment("Check if profiler or side effects check is enabled");
__ lbz(scratch,
__ ExternalReferenceAsOperand(IsolateFieldId::kExecutionMode));
__ cmpi(scratch, Operand::Zero());
__ bne(&profiler_or_side_effects_check_enabled);
#ifdef V8_RUNTIME_CALL_STATS
__ RecordComment("Check if RCS is enabled");
__ Move(scratch, ER::address_of_runtime_stats_flag());
__ lwz(scratch, MemOperand(scratch, 0));
__ cmpi(scratch, Operand::Zero());
__ bne(&profiler_or_side_effects_check_enabled);
#endif // V8_RUNTIME_CALL_STATS
}
__ RecordComment("Call the api function directly.");
__ StoreReturnAddressAndCall(function_address);
__ bind(&done_api_call);
if (handle_interceptor_result) {
// Skip reading return value if the callback returned kInterceptedNo,
// this would make the builtin return kNotInterceptedSentinel value.
// Size is important here, otherwise the C++ function could have returned
// one- or two-byte value with junk in the upper part.
static_assert(kInterceptedNo == 1 && kInterceptedSize == 4);
static_assert(kInterceptedNo == kNotInterceptedSentinel);
static_assert(kInterceptedYes == 0);
__ andi(scratch, return_value, Operand(1));
__ b(to_condition(kNotZero), &done_reading_result);
}
Label propagate_exception;
Label delete_allocated_handles;
Label leave_exit_frame;
// load value from ReturnValue
__ RecordComment("Load the value from ReturnValue");
__ LoadU64(r3, return_value_operand);
__ bind(&done_reading_result);
{
ASM_CODE_COMMENT_STRING(
masm,
"No more valid handles (the result handle was the last one)."
"Restore previous handle scope.");
__ StoreU64(prev_next_address_reg, next_mem_op);
if (v8_flags.debug_code) {
__ lwz(scratch, level_mem_op);
__ subi(scratch, scratch, Operand(1));
__ CmpS64(scratch, prev_level_reg);
__ Check(eq, AbortReason::kUnexpectedLevelAfterReturnFromApiCall);
}
__ stw(prev_level_reg, level_mem_op);
__ LoadU64(scratch, limit_mem_op);
__ CmpS64(scratch, prev_limit_reg);
__ bne(&delete_allocated_handles);
}
__ RecordComment("Leave the API exit frame.");
__ bind(&leave_exit_frame);
Register argc_reg = prev_limit_reg;
if (argc_operand != nullptr) {
// Load the number of stack slots to drop before LeaveExitFrame modifies sp.
__ LoadU64(argc_reg, *argc_operand);
}
__ LeaveExitFrame();
{
ASM_CODE_COMMENT_STRING(masm,
"Check if the function scheduled an exception.");
UseScratchRegisterScope temps2(masm);
Register scratch2 = temps2.Acquire();
__ LoadRoot(scratch, RootIndex::kTheHoleValue);
__ LoadU64(scratch2, __ ExternalReferenceAsOperand(
ER::exception_address(isolate), no_reg));
__ CmpS64(scratch, scratch2);
__ bne(&propagate_exception);
}
if (v8_flags.debug_code) {
Label ok;
if (handle_interceptor_result) {
__ Cmp(return_value, kNotInterceptedSentinel);
__ beq(&ok);
}
__ AssertJSAny(return_value, AbortReason::kAPICallReturnedInvalidObject);
__ bind(&ok);
}
if (argc_operand == nullptr) {
DCHECK_NE(slots_to_drop_on_return, 0);
__ AddS64(sp, sp, Operand(slots_to_drop_on_return * kSystemPointerSize));
} else {
// {argc_operand} was loaded into {argc_reg} above.
__ AddS64(sp, sp, Operand(slots_to_drop_on_return * kSystemPointerSize));
__ Drop(argc_reg);
}
__ blr();
if (with_profiling) {
ASM_CODE_COMMENT_STRING(masm, "Call the api function via the thunk.");
__ bind(&profiler_or_side_effects_check_enabled);
// Additional parameter if provided.
if (thunk_arg.is_valid()) {
MemOperand thunk_arg_mem_op = __ ExternalReferenceAsOperand(
IsolateFieldId::kApiCallbackThunkArgument);
__ StoreU64(thunk_arg, thunk_arg_mem_op);
}
__ Move(scratch, thunk_ref);
__ StoreReturnAddressAndCall(scratch);
__ b(&done_api_call);
}
__ RecordComment("An exception was thrown. Propagate it.");
__ bind(&propagate_exception);
__ TailCallRuntime(Runtime::kPropagateException);
{
ASM_CODE_COMMENT_STRING(
masm, "HandleScope limit has changed. Delete allocated extensions.");
__ bind(&delete_allocated_handles);
__ StoreU64(prev_limit_reg, limit_mem_op);
// Save the return value in a callee-save register.
Register saved_result = prev_limit_reg;
__ mr(saved_result, return_value);
__ PrepareCallCFunction(1);
__ Move(kCArgRegs[0], ER::isolate_address());
__ CallCFunction(ER::delete_handle_scope_extensions(), 1);
__ mr(return_value, saved_result);
__ b(&leave_exit_frame);
}
}
void MacroAssembler::Switch(Register scratch, Register value,
int case_value_base, Label** labels,
int num_labels) {
Label fallthrough, jump_table;
if (case_value_base != 0) {
SubS64(value, value, Operand(case_value_base));
}
CmpU64(value, Operand(num_labels));
bge(&fallthrough);
int entry_size_log2 = 2;
ShiftLeftU32(value, value, Operand(entry_size_log2));
Assembler::BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(!AreAliased(scratch, value, r0));
GetLabelAddress(scratch, &jump_table);
add(scratch, scratch, value);
Jump(scratch);
bind(&jump_table);
for (int i = 0; i < num_labels; ++i) {
b(labels[i]);
}
bind(&fallthrough);
}
} // namespace internal
} // namespace v8
#undef __
#endif // V8_TARGET_ARCH_PPC64