/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* This file contains codegen for the Thumb2 ISA. */
#include "codegen_arm64.h"
#include "arch/instruction_set_features.h"
#include "arm64_lir.h"
#include "base/bit_utils.h"
#include "base/logging.h"
#include "dex/compiler_ir.h"
#include "dex/mir_graph.h"
#include "dex/quick/mir_to_lir-inl.h"
#include "dex/reg_storage_eq.h"
#include "driver/compiler_driver.h"
#include "entrypoints/quick/quick_entrypoints.h"
#include "mirror/array-inl.h"
namespace art {
LIR* Arm64Mir2Lir::OpCmpBranch(ConditionCode cond, RegStorage src1, RegStorage src2, LIR* target) {
OpRegReg(kOpCmp, src1, src2);
return OpCondBranch(cond, target);
}
LIR* Arm64Mir2Lir::OpIT(ConditionCode ccode, const char* guide) {
UNUSED(ccode, guide);
LOG(FATAL) << "Unexpected use of OpIT for Arm64";
UNREACHABLE();
}
void Arm64Mir2Lir::OpEndIT(LIR* it) {
UNUSED(it);
LOG(FATAL) << "Unexpected use of OpEndIT for Arm64";
}
/*
* 64-bit 3way compare function.
* cmp xA, xB
* csinc wC, wzr, wzr, eq // wC = (xA == xB) ? 0 : 1
* csneg wC, wC, wC, ge // wC = (xA >= xB) ? wC : -wC
*/
void Arm64Mir2Lir::GenCmpLong(RegLocation rl_dest, RegLocation rl_src1,
RegLocation rl_src2) {
RegLocation rl_result;
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
rl_src2 = LoadValueWide(rl_src2, kCoreReg);
rl_result = EvalLoc(rl_dest, kCoreReg, true);
OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg);
NewLIR4(kA64Csinc4rrrc, rl_result.reg.GetReg(), rwzr, rwzr, kArmCondEq);
NewLIR4(kA64Csneg4rrrc, rl_result.reg.GetReg(), rl_result.reg.GetReg(),
rl_result.reg.GetReg(), kArmCondGe);
StoreValue(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenShiftOpLong(Instruction::Code opcode, RegLocation rl_dest,
RegLocation rl_src1, RegLocation rl_shift) {
OpKind op = kOpBkpt;
switch (opcode) {
case Instruction::SHL_LONG:
case Instruction::SHL_LONG_2ADDR:
op = kOpLsl;
break;
case Instruction::SHR_LONG:
case Instruction::SHR_LONG_2ADDR:
op = kOpAsr;
break;
case Instruction::USHR_LONG:
case Instruction::USHR_LONG_2ADDR:
op = kOpLsr;
break;
default:
LOG(FATAL) << "Unexpected case: " << opcode;
}
rl_shift = LoadValue(rl_shift, kCoreReg);
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true);
OpRegRegReg(op, rl_result.reg, rl_src1.reg, As64BitReg(rl_shift.reg));
StoreValueWide(rl_dest, rl_result);
}
static constexpr bool kUseDeltaEncodingInGenSelect = false;
void Arm64Mir2Lir::GenSelect(int32_t true_val, int32_t false_val, ConditionCode ccode,
RegStorage rs_dest, int result_reg_class) {
if (false_val == 0 || // 0 is better as first operand.
true_val == 1 || // Potentially Csinc.
true_val == -1 || // Potentially Csinv.
true_val == false_val + 1) { // Potentially Csinc.
ccode = NegateComparison(ccode);
std::swap(true_val, false_val);
}
ArmConditionCode code = ArmConditionEncoding(ccode);
int opcode; // The opcode.
RegStorage left_op = RegStorage::InvalidReg(); // The operands.
RegStorage right_op = RegStorage::InvalidReg(); // The operands.
bool is_wide = rs_dest.Is64Bit();
RegStorage zero_reg = is_wide ? rs_xzr : rs_wzr;
if (true_val == 0) {
left_op = zero_reg;
} else {
left_op = rs_dest;
LoadConstantNoClobber(rs_dest, true_val);
}
if (false_val == 1) {
right_op = zero_reg;
opcode = kA64Csinc4rrrc;
} else if (false_val == -1) {
right_op = zero_reg;
opcode = kA64Csinv4rrrc;
} else if (false_val == true_val + 1) {
right_op = left_op;
opcode = kA64Csinc4rrrc;
} else if (false_val == -true_val) {
right_op = left_op;
opcode = kA64Csneg4rrrc;
} else if (false_val == ~true_val) {
right_op = left_op;
opcode = kA64Csinv4rrrc;
} else if (true_val == 0) {
// left_op is zero_reg.
right_op = rs_dest;
LoadConstantNoClobber(rs_dest, false_val);
opcode = kA64Csel4rrrc;
} else {
// Generic case.
RegStorage t_reg2 = AllocTypedTemp(false, result_reg_class);
if (is_wide) {
if (t_reg2.Is32Bit()) {
t_reg2 = As64BitReg(t_reg2);
}
} else {
if (t_reg2.Is64Bit()) {
t_reg2 = As32BitReg(t_reg2);
}
}
if (kUseDeltaEncodingInGenSelect) {
int32_t delta = false_val - true_val;
uint32_t abs_val = delta < 0 ? -delta : delta;
if (abs_val < 0x1000) { // TODO: Replace with InexpensiveConstant with opcode.
// Can encode as immediate to an add.
right_op = t_reg2;
OpRegRegImm(kOpAdd, t_reg2, left_op, delta);
}
}
// Load as constant.
if (!right_op.Valid()) {
LoadConstantNoClobber(t_reg2, false_val);
right_op = t_reg2;
}
opcode = kA64Csel4rrrc;
}
DCHECK(left_op.Valid() && right_op.Valid());
NewLIR4(is_wide ? WIDE(opcode) : opcode, rs_dest.GetReg(), left_op.GetReg(), right_op.GetReg(),
code);
}
void Arm64Mir2Lir::GenSelectConst32(RegStorage left_op, RegStorage right_op, ConditionCode code,
int32_t true_val, int32_t false_val, RegStorage rs_dest,
RegisterClass dest_reg_class) {
DCHECK(rs_dest.Valid());
OpRegReg(kOpCmp, left_op, right_op);
GenSelect(true_val, false_val, code, rs_dest, dest_reg_class);
}
void Arm64Mir2Lir::GenSelect(BasicBlock* bb, MIR* mir) {
UNUSED(bb);
RegLocation rl_src = mir_graph_->GetSrc(mir, 0);
rl_src = LoadValue(rl_src, rl_src.ref ? kRefReg : kCoreReg);
// rl_src may be aliased with rl_result/rl_dest, so do compare early.
OpRegImm(kOpCmp, rl_src.reg, 0);
RegLocation rl_dest = mir_graph_->GetDest(mir);
// The kMirOpSelect has two variants, one for constants and one for moves.
if (mir->ssa_rep->num_uses == 1) {
RegLocation rl_result = EvalLoc(rl_dest, rl_dest.ref ? kRefReg : kCoreReg, true);
GenSelect(mir->dalvikInsn.vB, mir->dalvikInsn.vC, mir->meta.ccode, rl_result.reg,
rl_dest.ref ? kRefReg : kCoreReg);
StoreValue(rl_dest, rl_result);
} else {
RegLocation rl_true = mir_graph_->reg_location_[mir->ssa_rep->uses[1]];
RegLocation rl_false = mir_graph_->reg_location_[mir->ssa_rep->uses[2]];
RegisterClass result_reg_class = rl_dest.ref ? kRefReg : kCoreReg;
rl_true = LoadValue(rl_true, result_reg_class);
rl_false = LoadValue(rl_false, result_reg_class);
RegLocation rl_result = EvalLoc(rl_dest, result_reg_class, true);
bool is_wide = rl_dest.ref || rl_dest.wide;
int opcode = is_wide ? WIDE(kA64Csel4rrrc) : kA64Csel4rrrc;
NewLIR4(opcode, rl_result.reg.GetReg(),
rl_true.reg.GetReg(), rl_false.reg.GetReg(), ArmConditionEncoding(mir->meta.ccode));
StoreValue(rl_dest, rl_result);
}
}
void Arm64Mir2Lir::GenFusedLongCmpBranch(BasicBlock* bb, MIR* mir) {
RegLocation rl_src1 = mir_graph_->GetSrcWide(mir, 0);
RegLocation rl_src2 = mir_graph_->GetSrcWide(mir, 2);
LIR* taken = &block_label_list_[bb->taken];
LIR* not_taken = &block_label_list_[bb->fall_through];
// Normalize such that if either operand is constant, src2 will be constant.
ConditionCode ccode = mir->meta.ccode;
if (rl_src1.is_const) {
std::swap(rl_src1, rl_src2);
ccode = FlipComparisonOrder(ccode);
}
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
if (rl_src2.is_const) {
// TODO: Optimize for rl_src1.is_const? (Does happen in the boot image at the moment.)
int64_t val = mir_graph_->ConstantValueWide(rl_src2);
// Special handling using cbz & cbnz.
if (val == 0 && (ccode == kCondEq || ccode == kCondNe)) {
OpCmpImmBranch(ccode, rl_src1.reg, 0, taken);
OpCmpImmBranch(NegateComparison(ccode), rl_src1.reg, 0, not_taken);
return;
}
// Only handle Imm if src2 is not already in a register.
rl_src2 = UpdateLocWide(rl_src2);
if (rl_src2.location != kLocPhysReg) {
OpRegImm64(kOpCmp, rl_src1.reg, val);
OpCondBranch(ccode, taken);
OpCondBranch(NegateComparison(ccode), not_taken);
return;
}
}
rl_src2 = LoadValueWide(rl_src2, kCoreReg);
OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg);
OpCondBranch(ccode, taken);
OpCondBranch(NegateComparison(ccode), not_taken);
}
/*
* Generate a register comparison to an immediate and branch. Caller
* is responsible for setting branch target field.
*/
LIR* Arm64Mir2Lir::OpCmpImmBranch(ConditionCode cond, RegStorage reg, int check_value,
LIR* target) {
LIR* branch = nullptr;
ArmConditionCode arm_cond = ArmConditionEncoding(cond);
if (check_value == 0) {
if (arm_cond == kArmCondEq || arm_cond == kArmCondNe) {
A64Opcode opcode = (arm_cond == kArmCondEq) ? kA64Cbz2rt : kA64Cbnz2rt;
A64Opcode wide = reg.Is64Bit() ? WIDE(0) : UNWIDE(0);
branch = NewLIR2(opcode | wide, reg.GetReg(), 0);
} else if (arm_cond == kArmCondLs) {
// kArmCondLs is an unsigned less or equal. A comparison r <= 0 is then the same as cbz.
// This case happens for a bounds check of array[0].
A64Opcode opcode = kA64Cbz2rt;
A64Opcode wide = reg.Is64Bit() ? WIDE(0) : UNWIDE(0);
branch = NewLIR2(opcode | wide, reg.GetReg(), 0);
} else if (arm_cond == kArmCondLt || arm_cond == kArmCondGe) {
A64Opcode opcode = (arm_cond == kArmCondLt) ? kA64Tbnz3rht : kA64Tbz3rht;
A64Opcode wide = reg.Is64Bit() ? WIDE(0) : UNWIDE(0);
int value = reg.Is64Bit() ? 63 : 31;
branch = NewLIR3(opcode | wide, reg.GetReg(), value, 0);
}
}
if (branch == nullptr) {
OpRegImm(kOpCmp, reg, check_value);
branch = NewLIR2(kA64B2ct, arm_cond, 0);
}
branch->target = target;
return branch;
}
LIR* Arm64Mir2Lir::OpCmpMemImmBranch(ConditionCode cond, RegStorage temp_reg,
RegStorage base_reg, int offset, int check_value,
LIR* target, LIR** compare) {
DCHECK(compare == nullptr);
// It is possible that temp register is 64-bit. (ArgReg or RefReg)
// Always compare 32-bit value no matter what temp_reg is.
if (temp_reg.Is64Bit()) {
temp_reg = As32BitReg(temp_reg);
}
Load32Disp(base_reg, offset, temp_reg);
LIR* branch = OpCmpImmBranch(cond, temp_reg, check_value, target);
return branch;
}
LIR* Arm64Mir2Lir::OpRegCopyNoInsert(RegStorage r_dest, RegStorage r_src) {
bool dest_is_fp = r_dest.IsFloat();
bool src_is_fp = r_src.IsFloat();
A64Opcode opcode = kA64Brk1d;
LIR* res;
if (LIKELY(dest_is_fp == src_is_fp)) {
if (LIKELY(!dest_is_fp)) {
DCHECK_EQ(r_dest.Is64Bit(), r_src.Is64Bit());
// Core/core copy.
// Copies involving the sp register require a different instruction.
opcode = UNLIKELY(A64_REG_IS_SP(r_dest.GetReg())) ? kA64Add4RRdT : kA64Mov2rr;
// TODO(Arm64): kA64Add4RRdT formally has 4 args, but is used as a 2 args instruction.
// This currently works because the other arguments are set to 0 by default. We should
// rather introduce an alias kA64Mov2RR.
// core/core copy. Do a x/x copy only if both registers are x.
if (r_dest.Is64Bit() && r_src.Is64Bit()) {
opcode = WIDE(opcode);
}
} else {
// Float/float copy.
bool dest_is_double = r_dest.IsDouble();
bool src_is_double = r_src.IsDouble();
// We do not do float/double or double/float casts here.
DCHECK_EQ(dest_is_double, src_is_double);
// Homogeneous float/float copy.
opcode = (dest_is_double) ? WIDE(kA64Fmov2ff) : kA64Fmov2ff;
}
} else {
// Inhomogeneous register copy.
if (dest_is_fp) {
if (r_dest.IsDouble()) {
opcode = kA64Fmov2Sx;
} else {
r_src = Check32BitReg(r_src);
opcode = kA64Fmov2sw;
}
} else {
if (r_src.IsDouble()) {
opcode = kA64Fmov2xS;
} else {
r_dest = Check32BitReg(r_dest);
opcode = kA64Fmov2ws;
}
}
}
res = RawLIR(current_dalvik_offset_, opcode, r_dest.GetReg(), r_src.GetReg());
if (!(cu_->disable_opt & (1 << kSafeOptimizations)) && r_dest == r_src) {
res->flags.is_nop = true;
}
return res;
}
void Arm64Mir2Lir::OpRegCopy(RegStorage r_dest, RegStorage r_src) {
if (r_dest != r_src) {
LIR* res = OpRegCopyNoInsert(r_dest, r_src);
AppendLIR(res);
}
}
void Arm64Mir2Lir::OpRegCopyWide(RegStorage r_dest, RegStorage r_src) {
OpRegCopy(r_dest, r_src);
}
// Table of magic divisors
struct MagicTable {
int magic64_base;
int magic64_eor;
uint64_t magic64;
uint32_t magic32;
uint32_t shift;
DividePattern pattern;
};
static const MagicTable magic_table[] = {
{ 0, 0, 0, 0, 0, DivideNone}, // 0
{ 0, 0, 0, 0, 0, DivideNone}, // 1
{ 0, 0, 0, 0, 0, DivideNone}, // 2
{0x3c, -1, 0x5555555555555556, 0x55555556, 0, Divide3}, // 3
{ 0, 0, 0, 0, 0, DivideNone}, // 4
{0xf9, -1, 0x6666666666666667, 0x66666667, 1, Divide5}, // 5
{0x7c, 0x1041, 0x2AAAAAAAAAAAAAAB, 0x2AAAAAAB, 0, Divide3}, // 6
{ -1, -1, 0x924924924924924A, 0x92492493, 2, Divide7}, // 7
{ 0, 0, 0, 0, 0, DivideNone}, // 8
{ -1, -1, 0x38E38E38E38E38E4, 0x38E38E39, 1, Divide5}, // 9
{0xf9, -1, 0x6666666666666667, 0x66666667, 2, Divide5}, // 10
{ -1, -1, 0x2E8BA2E8BA2E8BA3, 0x2E8BA2E9, 1, Divide5}, // 11
{0x7c, 0x1041, 0x2AAAAAAAAAAAAAAB, 0x2AAAAAAB, 1, Divide5}, // 12
{ -1, -1, 0x4EC4EC4EC4EC4EC5, 0x4EC4EC4F, 2, Divide5}, // 13
{ -1, -1, 0x924924924924924A, 0x92492493, 3, Divide7}, // 14
{0x78, -1, 0x8888888888888889, 0x88888889, 3, Divide7}, // 15
};
// Integer division by constant via reciprocal multiply (Hacker's Delight, 10-4)
bool Arm64Mir2Lir::SmallLiteralDivRem(Instruction::Code dalvik_opcode, bool is_div,
RegLocation rl_src, RegLocation rl_dest, int lit) {
UNUSED(dalvik_opcode);
if ((lit < 0) || (lit >= static_cast<int>(arraysize(magic_table)))) {
return false;
}
DividePattern pattern = magic_table[lit].pattern;
if (pattern == DivideNone) {
return false;
}
// Tuning: add rem patterns
if (!is_div) {
return false;
}
RegStorage r_magic = AllocTemp();
LoadConstant(r_magic, magic_table[lit].magic32);
rl_src = LoadValue(rl_src, kCoreReg);
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
RegStorage r_long_mul = AllocTemp();
NewLIR3(kA64Smull3xww, As64BitReg(r_long_mul).GetReg(), r_magic.GetReg(), rl_src.reg.GetReg());
switch (pattern) {
case Divide3:
OpRegRegImm(kOpLsr, As64BitReg(r_long_mul), As64BitReg(r_long_mul), 32);
OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 31));
break;
case Divide5:
OpRegRegImm(kOpAsr, As64BitReg(r_long_mul), As64BitReg(r_long_mul),
32 + magic_table[lit].shift);
OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 31));
break;
case Divide7:
OpRegRegRegShift(kOpAdd, As64BitReg(r_long_mul), As64BitReg(rl_src.reg),
As64BitReg(r_long_mul), EncodeShift(kA64Lsr, 32));
OpRegRegImm(kOpAsr, r_long_mul, r_long_mul, magic_table[lit].shift);
OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 31));
break;
default:
LOG(FATAL) << "Unexpected pattern: " << pattern;
}
StoreValue(rl_dest, rl_result);
return true;
}
bool Arm64Mir2Lir::SmallLiteralDivRem64(Instruction::Code dalvik_opcode, bool is_div,
RegLocation rl_src, RegLocation rl_dest, int64_t lit) {
UNUSED(dalvik_opcode);
if ((lit < 0) || (lit >= static_cast<int>(arraysize(magic_table)))) {
return false;
}
DividePattern pattern = magic_table[lit].pattern;
if (pattern == DivideNone) {
return false;
}
// Tuning: add rem patterns
if (!is_div) {
return false;
}
RegStorage r_magic = AllocTempWide();
rl_src = LoadValueWide(rl_src, kCoreReg);
RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true);
RegStorage r_long_mul = AllocTempWide();
if (magic_table[lit].magic64_base >= 0) {
// Check that the entry in the table is correct.
if (kIsDebugBuild) {
uint64_t reconstructed_imm;
uint64_t base = DecodeLogicalImmediate(/*is_wide*/true, magic_table[lit].magic64_base);
if (magic_table[lit].magic64_eor >= 0) {
uint64_t eor = DecodeLogicalImmediate(/*is_wide*/true, magic_table[lit].magic64_eor);
reconstructed_imm = base ^ eor;
} else {
reconstructed_imm = base + 1;
}
DCHECK_EQ(reconstructed_imm, magic_table[lit].magic64) << " for literal " << lit;
}
// Load the magic constant in two instructions.
NewLIR3(WIDE(kA64Orr3Rrl), r_magic.GetReg(), rxzr, magic_table[lit].magic64_base);
if (magic_table[lit].magic64_eor >= 0) {
NewLIR3(WIDE(kA64Eor3Rrl), r_magic.GetReg(), r_magic.GetReg(),
magic_table[lit].magic64_eor);
} else {
NewLIR4(WIDE(kA64Add4RRdT), r_magic.GetReg(), r_magic.GetReg(), 1, 0);
}
} else {
LoadConstantWide(r_magic, magic_table[lit].magic64);
}
NewLIR3(kA64Smulh3xxx, r_long_mul.GetReg(), r_magic.GetReg(), rl_src.reg.GetReg());
switch (pattern) {
case Divide3:
OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 63));
break;
case Divide5:
OpRegRegImm(kOpAsr, r_long_mul, r_long_mul, magic_table[lit].shift);
OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 63));
break;
case Divide7:
OpRegRegReg(kOpAdd, r_long_mul, rl_src.reg, r_long_mul);
OpRegRegImm(kOpAsr, r_long_mul, r_long_mul, magic_table[lit].shift);
OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 63));
break;
default:
LOG(FATAL) << "Unexpected pattern: " << pattern;
}
StoreValueWide(rl_dest, rl_result);
return true;
}
// Returns true if it added instructions to 'cu' to divide 'rl_src' by 'lit'
// and store the result in 'rl_dest'.
bool Arm64Mir2Lir::HandleEasyDivRem(Instruction::Code dalvik_opcode, bool is_div,
RegLocation rl_src, RegLocation rl_dest, int lit) {
return HandleEasyDivRem64(dalvik_opcode, is_div, rl_src, rl_dest, static_cast<int>(lit));
}
// Returns true if it added instructions to 'cu' to divide 'rl_src' by 'lit'
// and store the result in 'rl_dest'.
bool Arm64Mir2Lir::HandleEasyDivRem64(Instruction::Code dalvik_opcode, bool is_div,
RegLocation rl_src, RegLocation rl_dest, int64_t lit) {
const bool is_64bit = rl_dest.wide;
const int nbits = (is_64bit) ? 64 : 32;
if (lit < 2) {
return false;
}
if (!IsPowerOfTwo(lit)) {
if (is_64bit) {
return SmallLiteralDivRem64(dalvik_opcode, is_div, rl_src, rl_dest, lit);
} else {
return SmallLiteralDivRem(dalvik_opcode, is_div, rl_src, rl_dest, static_cast<int32_t>(lit));
}
}
int k = CTZ(lit);
if (k >= nbits - 2) {
// Avoid special cases.
return false;
}
RegLocation rl_result;
RegStorage t_reg;
if (is_64bit) {
rl_src = LoadValueWide(rl_src, kCoreReg);
rl_result = EvalLocWide(rl_dest, kCoreReg, true);
t_reg = AllocTempWide();
} else {
rl_src = LoadValue(rl_src, kCoreReg);
rl_result = EvalLoc(rl_dest, kCoreReg, true);
t_reg = AllocTemp();
}
int shift = EncodeShift(kA64Lsr, nbits - k);
if (is_div) {
if (lit == 2) {
// Division by 2 is by far the most common division by constant.
OpRegRegRegShift(kOpAdd, t_reg, rl_src.reg, rl_src.reg, shift);
OpRegRegImm(kOpAsr, rl_result.reg, t_reg, k);
} else {
OpRegRegImm(kOpAsr, t_reg, rl_src.reg, nbits - 1);
OpRegRegRegShift(kOpAdd, t_reg, rl_src.reg, t_reg, shift);
OpRegRegImm(kOpAsr, rl_result.reg, t_reg, k);
}
} else {
if (lit == 2) {
OpRegRegRegShift(kOpAdd, t_reg, rl_src.reg, rl_src.reg, shift);
OpRegRegImm64(kOpAnd, t_reg, t_reg, lit - 1);
OpRegRegRegShift(kOpSub, rl_result.reg, t_reg, rl_src.reg, shift);
} else {
RegStorage t_reg2 = (is_64bit) ? AllocTempWide() : AllocTemp();
OpRegRegImm(kOpAsr, t_reg, rl_src.reg, nbits - 1);
OpRegRegRegShift(kOpAdd, t_reg2, rl_src.reg, t_reg, shift);
OpRegRegImm64(kOpAnd, t_reg2, t_reg2, lit - 1);
OpRegRegRegShift(kOpSub, rl_result.reg, t_reg2, t_reg, shift);
}
}
if (is_64bit) {
StoreValueWide(rl_dest, rl_result);
} else {
StoreValue(rl_dest, rl_result);
}
return true;
}
bool Arm64Mir2Lir::EasyMultiply(RegLocation rl_src, RegLocation rl_dest, int lit) {
UNUSED(rl_src, rl_dest, lit);
LOG(FATAL) << "Unexpected use of EasyMultiply for Arm64";
UNREACHABLE();
}
RegLocation Arm64Mir2Lir::GenDivRemLit(RegLocation rl_dest, RegLocation rl_src1, int lit,
bool is_div) {
UNUSED(rl_dest, rl_src1, lit, is_div);
LOG(FATAL) << "Unexpected use of GenDivRemLit for Arm64";
UNREACHABLE();
}
RegLocation Arm64Mir2Lir::GenDivRemLit(RegLocation rl_dest, RegStorage reg1, int lit, bool is_div) {
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
// Put the literal in a temp.
RegStorage lit_temp = AllocTemp();
LoadConstant(lit_temp, lit);
// Use the generic case for div/rem with arg2 in a register.
// TODO: The literal temp can be freed earlier during a modulus to reduce reg pressure.
rl_result = GenDivRem(rl_result, reg1, lit_temp, is_div);
FreeTemp(lit_temp);
return rl_result;
}
RegLocation Arm64Mir2Lir::GenDivRem(RegLocation rl_dest, RegLocation rl_src1,
RegLocation rl_src2, bool is_div, int flags) {
UNUSED(rl_dest, rl_src1, rl_src2, is_div, flags);
LOG(FATAL) << "Unexpected use of GenDivRem for Arm64";
UNREACHABLE();
}
RegLocation Arm64Mir2Lir::GenDivRem(RegLocation rl_dest, RegStorage r_src1, RegStorage r_src2,
bool is_div) {
CHECK_EQ(r_src1.Is64Bit(), r_src2.Is64Bit());
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
if (is_div) {
OpRegRegReg(kOpDiv, rl_result.reg, r_src1, r_src2);
} else {
// temp = r_src1 / r_src2
// dest = r_src1 - temp * r_src2
RegStorage temp;
A64Opcode wide;
if (rl_result.reg.Is64Bit()) {
temp = AllocTempWide();
wide = WIDE(0);
} else {
temp = AllocTemp();
wide = UNWIDE(0);
}
OpRegRegReg(kOpDiv, temp, r_src1, r_src2);
NewLIR4(kA64Msub4rrrr | wide, rl_result.reg.GetReg(), temp.GetReg(),
r_src2.GetReg(), r_src1.GetReg());
FreeTemp(temp);
}
return rl_result;
}
bool Arm64Mir2Lir::GenInlinedAbsInt(CallInfo* info) {
RegLocation rl_src = info->args[0];
rl_src = LoadValue(rl_src, kCoreReg);
RegLocation rl_dest = InlineTarget(info);
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
// Compare the source value with zero. Write the negated value to the result if
// negative, otherwise write the original value.
OpRegImm(kOpCmp, rl_src.reg, 0);
NewLIR4(kA64Csneg4rrrc, rl_result.reg.GetReg(), rl_src.reg.GetReg(), rl_src.reg.GetReg(),
kArmCondPl);
StoreValue(rl_dest, rl_result);
return true;
}
bool Arm64Mir2Lir::GenInlinedAbsLong(CallInfo* info) {
RegLocation rl_src = info->args[0];
rl_src = LoadValueWide(rl_src, kCoreReg);
RegLocation rl_dest = InlineTargetWide(info);
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
// Compare the source value with zero. Write the negated value to the result if
// negative, otherwise write the original value.
OpRegImm(kOpCmp, rl_src.reg, 0);
NewLIR4(WIDE(kA64Csneg4rrrc), rl_result.reg.GetReg(), rl_src.reg.GetReg(),
rl_src.reg.GetReg(), kArmCondPl);
StoreValueWide(rl_dest, rl_result);
return true;
}
bool Arm64Mir2Lir::GenInlinedMinMax(CallInfo* info, bool is_min, bool is_long) {
DCHECK_EQ(cu_->instruction_set, kArm64);
RegLocation rl_src1 = info->args[0];
RegLocation rl_src2 = (is_long) ? info->args[2] : info->args[1];
rl_src1 = (is_long) ? LoadValueWide(rl_src1, kCoreReg) : LoadValue(rl_src1, kCoreReg);
rl_src2 = (is_long) ? LoadValueWide(rl_src2, kCoreReg) : LoadValue(rl_src2, kCoreReg);
RegLocation rl_dest = (is_long) ? InlineTargetWide(info) : InlineTarget(info);
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg);
NewLIR4((is_long) ? WIDE(kA64Csel4rrrc) : kA64Csel4rrrc, rl_result.reg.GetReg(),
rl_src1.reg.GetReg(), rl_src2.reg.GetReg(), (is_min) ? kArmCondLt : kArmCondGt);
(is_long) ? StoreValueWide(rl_dest, rl_result) :StoreValue(rl_dest, rl_result);
return true;
}
bool Arm64Mir2Lir::GenInlinedPeek(CallInfo* info, OpSize size) {
RegLocation rl_src_address = info->args[0]; // long address
RegLocation rl_dest = (size == k64) ? InlineTargetWide(info) : InlineTarget(info);
RegLocation rl_address = LoadValueWide(rl_src_address, kCoreReg);
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
LoadBaseDisp(rl_address.reg, 0, rl_result.reg, size, kNotVolatile);
if (size == k64) {
StoreValueWide(rl_dest, rl_result);
} else {
DCHECK(size == kSignedByte || size == kSignedHalf || size == k32);
StoreValue(rl_dest, rl_result);
}
return true;
}
bool Arm64Mir2Lir::GenInlinedPoke(CallInfo* info, OpSize size) {
RegLocation rl_src_address = info->args[0]; // long address
RegLocation rl_src_value = info->args[2]; // [size] value
RegLocation rl_address = LoadValueWide(rl_src_address, kCoreReg);
RegLocation rl_value;
if (size == k64) {
rl_value = LoadValueWide(rl_src_value, kCoreReg);
} else {
DCHECK(size == kSignedByte || size == kSignedHalf || size == k32);
rl_value = LoadValue(rl_src_value, kCoreReg);
}
StoreBaseDisp(rl_address.reg, 0, rl_value.reg, size, kNotVolatile);
return true;
}
bool Arm64Mir2Lir::GenInlinedCas(CallInfo* info, bool is_long, bool is_object) {
DCHECK_EQ(cu_->instruction_set, kArm64);
// Unused - RegLocation rl_src_unsafe = info->args[0];
RegLocation rl_src_obj = info->args[1]; // Object - known non-null
RegLocation rl_src_offset = info->args[2]; // long low
RegLocation rl_src_expected = info->args[4]; // int, long or Object
// If is_long, high half is in info->args[5]
RegLocation rl_src_new_value = info->args[is_long ? 6 : 5]; // int, long or Object
// If is_long, high half is in info->args[7]
RegLocation rl_dest = InlineTarget(info); // boolean place for result
// Load Object and offset
RegLocation rl_object = LoadValue(rl_src_obj, kRefReg);
RegLocation rl_offset = LoadValueWide(rl_src_offset, kCoreReg);
RegLocation rl_new_value;
RegLocation rl_expected;
if (is_long) {
rl_new_value = LoadValueWide(rl_src_new_value, kCoreReg);
rl_expected = LoadValueWide(rl_src_expected, kCoreReg);
} else {
rl_new_value = LoadValue(rl_src_new_value, is_object ? kRefReg : kCoreReg);
rl_expected = LoadValue(rl_src_expected, is_object ? kRefReg : kCoreReg);
}
if (is_object && !mir_graph_->IsConstantNullRef(rl_new_value)) {
// Mark card for object assuming new value is stored.
MarkGCCard(0, rl_new_value.reg, rl_object.reg);
}
RegStorage r_ptr = AllocTempRef();
OpRegRegReg(kOpAdd, r_ptr, rl_object.reg, rl_offset.reg);
// Free now unneeded rl_object and rl_offset to give more temps.
ClobberSReg(rl_object.s_reg_low);
FreeTemp(rl_object.reg);
ClobberSReg(rl_offset.s_reg_low);
FreeTemp(rl_offset.reg);
// do {
// tmp = [r_ptr] - expected;
// } while (tmp == 0 && failure([r_ptr] <- r_new_value));
// result = tmp != 0;
RegStorage r_tmp;
RegStorage r_tmp_stored;
RegStorage rl_new_value_stored = rl_new_value.reg;
A64Opcode wide = UNWIDE(0);
if (is_long) {
r_tmp_stored = r_tmp = AllocTempWide();
wide = WIDE(0);
} else if (is_object) {
// References use 64-bit registers, but are stored as compressed 32-bit values.
// This means r_tmp_stored != r_tmp.
r_tmp = AllocTempRef();
r_tmp_stored = As32BitReg(r_tmp);
rl_new_value_stored = As32BitReg(rl_new_value_stored);
} else {
r_tmp_stored = r_tmp = AllocTemp();
}
RegStorage r_tmp32 = (r_tmp.Is32Bit()) ? r_tmp : As32BitReg(r_tmp);
LIR* loop = NewLIR0(kPseudoTargetLabel);
NewLIR2(kA64Ldaxr2rX | wide, r_tmp_stored.GetReg(), r_ptr.GetReg());
OpRegReg(kOpCmp, r_tmp, rl_expected.reg);
DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode));
LIR* early_exit = OpCondBranch(kCondNe, nullptr);
NewLIR3(kA64Stlxr3wrX | wide, r_tmp32.GetReg(), rl_new_value_stored.GetReg(), r_ptr.GetReg());
NewLIR3(kA64Cmp3RdT, r_tmp32.GetReg(), 0, ENCODE_NO_SHIFT);
DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode));
OpCondBranch(kCondNe, loop);
LIR* exit_loop = NewLIR0(kPseudoTargetLabel);
early_exit->target = exit_loop;
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
NewLIR4(kA64Csinc4rrrc, rl_result.reg.GetReg(), rwzr, rwzr, kArmCondNe);
FreeTemp(r_tmp); // Now unneeded.
FreeTemp(r_ptr); // Now unneeded.
StoreValue(rl_dest, rl_result);
return true;
}
bool Arm64Mir2Lir::GenInlinedArrayCopyCharArray(CallInfo* info) {
constexpr int kLargeArrayThreshold = 512;
RegLocation rl_src = info->args[0];
RegLocation rl_src_pos = info->args[1];
RegLocation rl_dst = info->args[2];
RegLocation rl_dst_pos = info->args[3];
RegLocation rl_length = info->args[4];
// Compile time check, handle exception by non-inline method to reduce related meta-data.
if ((rl_src_pos.is_const && (mir_graph_->ConstantValue(rl_src_pos) < 0)) ||
(rl_dst_pos.is_const && (mir_graph_->ConstantValue(rl_dst_pos) < 0)) ||
(rl_length.is_const && (mir_graph_->ConstantValue(rl_length) < 0))) {
return false;
}
ClobberCallerSave();
LockCallTemps(); // Prepare for explicit register usage.
RegStorage rs_src = rs_x0;
RegStorage rs_dst = rs_x1;
LoadValueDirectFixed(rl_src, rs_src);
LoadValueDirectFixed(rl_dst, rs_dst);
// Handle null pointer exception in slow-path.
LIR* src_check_branch = OpCmpImmBranch(kCondEq, rs_src, 0, nullptr);
LIR* dst_check_branch = OpCmpImmBranch(kCondEq, rs_dst, 0, nullptr);
// Handle potential overlapping in slow-path.
// TUNING: Support overlapping cases.
LIR* src_dst_same = OpCmpBranch(kCondEq, rs_src, rs_dst, nullptr);
// Handle exception or big length in slow-path.
RegStorage rs_length = rs_w2;
LoadValueDirectFixed(rl_length, rs_length);
LIR* len_neg_or_too_big = OpCmpImmBranch(kCondHi, rs_length, kLargeArrayThreshold, nullptr);
// Src bounds check.
RegStorage rs_src_pos = rs_w3;
RegStorage rs_arr_length = rs_w4;
LoadValueDirectFixed(rl_src_pos, rs_src_pos);
LIR* src_pos_negative = OpCmpImmBranch(kCondLt, rs_src_pos, 0, nullptr);
Load32Disp(rs_src, mirror::Array::LengthOffset().Int32Value(), rs_arr_length);
OpRegReg(kOpSub, rs_arr_length, rs_src_pos);
LIR* src_bad_len = OpCmpBranch(kCondLt, rs_arr_length, rs_length, nullptr);
// Dst bounds check.
RegStorage rs_dst_pos = rs_w5;
LoadValueDirectFixed(rl_dst_pos, rs_dst_pos);
LIR* dst_pos_negative = OpCmpImmBranch(kCondLt, rs_dst_pos, 0, nullptr);
Load32Disp(rs_dst, mirror::Array::LengthOffset().Int32Value(), rs_arr_length);
OpRegReg(kOpSub, rs_arr_length, rs_dst_pos);
LIR* dst_bad_len = OpCmpBranch(kCondLt, rs_arr_length, rs_length, nullptr);
// Everything is checked now.
// Set rs_src to the address of the first element to be copied.
rs_src_pos = As64BitReg(rs_src_pos);
OpRegImm(kOpAdd, rs_src, mirror::Array::DataOffset(2).Int32Value());
OpRegRegImm(kOpLsl, rs_src_pos, rs_src_pos, 1);
OpRegReg(kOpAdd, rs_src, rs_src_pos);
// Set rs_src to the address of the first element to be copied.
rs_dst_pos = As64BitReg(rs_dst_pos);
OpRegImm(kOpAdd, rs_dst, mirror::Array::DataOffset(2).Int32Value());
OpRegRegImm(kOpLsl, rs_dst_pos, rs_dst_pos, 1);
OpRegReg(kOpAdd, rs_dst, rs_dst_pos);
// rs_arr_length won't be not used anymore.
RegStorage rs_tmp = rs_arr_length;
// Use 64-bit view since rs_length will be used as index.
rs_length = As64BitReg(rs_length);
OpRegRegImm(kOpLsl, rs_length, rs_length, 1);
// Copy one element.
LIR* jmp_to_copy_two = NewLIR3(WIDE(kA64Tbz3rht), rs_length.GetReg(), 1, 0);
OpRegImm(kOpSub, rs_length, 2);
LoadBaseIndexed(rs_src, rs_length, rs_tmp, 0, kSignedHalf);
StoreBaseIndexed(rs_dst, rs_length, rs_tmp, 0, kSignedHalf);
// Copy two elements.
LIR *copy_two = NewLIR0(kPseudoTargetLabel);
LIR* jmp_to_copy_four = NewLIR3(WIDE(kA64Tbz3rht), rs_length.GetReg(), 2, 0);
OpRegImm(kOpSub, rs_length, 4);
LoadBaseIndexed(rs_src, rs_length, rs_tmp, 0, k32);
StoreBaseIndexed(rs_dst, rs_length, rs_tmp, 0, k32);
// Copy four elements.
LIR *copy_four = NewLIR0(kPseudoTargetLabel);
LIR* jmp_to_ret = OpCmpImmBranch(kCondEq, rs_length, 0, nullptr);
LIR *begin_loop = NewLIR0(kPseudoTargetLabel);
OpRegImm(kOpSub, rs_length, 8);
rs_tmp = As64BitReg(rs_tmp);
LoadBaseIndexed(rs_src, rs_length, rs_tmp, 0, k64);
StoreBaseIndexed(rs_dst, rs_length, rs_tmp, 0, k64);
LIR* jmp_to_loop = OpCmpImmBranch(kCondNe, rs_length, 0, nullptr);
LIR* loop_finished = OpUnconditionalBranch(nullptr);
LIR *check_failed = NewLIR0(kPseudoTargetLabel);
LIR* launchpad_branch = OpUnconditionalBranch(nullptr);
LIR* return_point = NewLIR0(kPseudoTargetLabel);
src_check_branch->target = check_failed;
dst_check_branch->target = check_failed;
src_dst_same->target = check_failed;
len_neg_or_too_big->target = check_failed;
src_pos_negative->target = check_failed;
src_bad_len->target = check_failed;
dst_pos_negative->target = check_failed;
dst_bad_len->target = check_failed;
jmp_to_copy_two->target = copy_two;
jmp_to_copy_four->target = copy_four;
jmp_to_ret->target = return_point;
jmp_to_loop->target = begin_loop;
loop_finished->target = return_point;
AddIntrinsicSlowPath(info, launchpad_branch, return_point);
ClobberCallerSave(); // We must clobber everything because slow path will return here
return true;
}
void Arm64Mir2Lir::OpPcRelLoad(RegStorage reg, LIR* target) {
ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral);
LIR* lir = NewLIR2(kA64Ldr2rp, As32BitReg(reg).GetReg(), 0);
lir->target = target;
}
bool Arm64Mir2Lir::CanUseOpPcRelDexCacheArrayLoad() const {
return dex_cache_arrays_layout_.Valid();
}
void Arm64Mir2Lir::OpPcRelDexCacheArrayLoad(const DexFile* dex_file, int offset, RegStorage r_dest,
bool wide) {
LIR* adrp = NewLIR2(kA64Adrp2xd, r_dest.GetReg(), 0);
adrp->operands[2] = WrapPointer(dex_file);
adrp->operands[3] = offset;
adrp->operands[4] = WrapPointer(adrp);
dex_cache_access_insns_.push_back(adrp);
if (wide) {
DCHECK(r_dest.Is64Bit());
}
LIR* ldr = LoadBaseDisp(r_dest, 0, r_dest, wide ? k64 : kReference, kNotVolatile);
ldr->operands[4] = adrp->operands[4];
ldr->flags.fixup = kFixupLabel;
dex_cache_access_insns_.push_back(ldr);
}
LIR* Arm64Mir2Lir::OpVldm(RegStorage r_base, int count) {
UNUSED(r_base, count);
LOG(FATAL) << "Unexpected use of OpVldm for Arm64";
UNREACHABLE();
}
LIR* Arm64Mir2Lir::OpVstm(RegStorage r_base, int count) {
UNUSED(r_base, count);
LOG(FATAL) << "Unexpected use of OpVstm for Arm64";
UNREACHABLE();
}
void Arm64Mir2Lir::GenMaddMsubInt(RegLocation rl_dest, RegLocation rl_src1, RegLocation rl_src2,
RegLocation rl_src3, bool is_sub) {
rl_src1 = LoadValue(rl_src1, kCoreReg);
rl_src2 = LoadValue(rl_src2, kCoreReg);
rl_src3 = LoadValue(rl_src3, kCoreReg);
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
NewLIR4(is_sub ? kA64Msub4rrrr : kA64Madd4rrrr, rl_result.reg.GetReg(), rl_src1.reg.GetReg(),
rl_src2.reg.GetReg(), rl_src3.reg.GetReg());
StoreValue(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenMaddMsubLong(RegLocation rl_dest, RegLocation rl_src1, RegLocation rl_src2,
RegLocation rl_src3, bool is_sub) {
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
rl_src2 = LoadValueWide(rl_src2, kCoreReg);
rl_src3 = LoadValueWide(rl_src3, kCoreReg);
RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true);
NewLIR4(is_sub ? WIDE(kA64Msub4rrrr) : WIDE(kA64Madd4rrrr), rl_result.reg.GetReg(),
rl_src1.reg.GetReg(), rl_src2.reg.GetReg(), rl_src3.reg.GetReg());
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenMultiplyByTwoBitMultiplier(RegLocation rl_src,
RegLocation rl_result, int lit ATTRIBUTE_UNUSED,
int first_bit, int second_bit) {
OpRegRegRegShift(kOpAdd, rl_result.reg, rl_src.reg, rl_src.reg,
EncodeShift(kA64Lsl, second_bit - first_bit));
if (first_bit != 0) {
OpRegRegImm(kOpLsl, rl_result.reg, rl_result.reg, first_bit);
}
}
void Arm64Mir2Lir::GenDivZeroCheckWide(RegStorage reg ATTRIBUTE_UNUSED) {
LOG(FATAL) << "Unexpected use of GenDivZero for Arm64";
}
// Test suspend flag, return target of taken suspend branch
LIR* Arm64Mir2Lir::OpTestSuspend(LIR* target) {
RegStorage r_tmp = AllocTemp();
LoadBaseDisp(rs_xSELF, Thread::ThreadFlagsOffset<kArm64PointerSize>().Int32Value(), r_tmp,
kUnsignedHalf, kNotVolatile);
LIR* cmp_branch = OpCmpImmBranch(target == nullptr ? kCondNe: kCondEq, r_tmp, 0, target);
FreeTemp(r_tmp);
return cmp_branch;
}
// Decrement register and branch on condition
LIR* Arm64Mir2Lir::OpDecAndBranch(ConditionCode c_code, RegStorage reg, LIR* target) {
// Combine sub & test using sub setflags encoding here. We need to make sure a
// subtract form that sets carry is used, so generate explicitly.
// TODO: might be best to add a new op, kOpSubs, and handle it generically.
A64Opcode opcode = reg.Is64Bit() ? WIDE(kA64Subs3rRd) : UNWIDE(kA64Subs3rRd);
NewLIR3(opcode, reg.GetReg(), reg.GetReg(), 1); // For value == 1, this should set flags.
DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode));
return OpCondBranch(c_code, target);
}
bool Arm64Mir2Lir::GenMemBarrier(MemBarrierKind barrier_kind) {
if (!cu_->compiler_driver->GetInstructionSetFeatures()->IsSmp()) {
return false;
}
// Start off with using the last LIR as the barrier. If it is not enough, then we will generate one.
LIR* barrier = last_lir_insn_;
int dmb_flavor;
// TODO: revisit Arm barrier kinds
switch (barrier_kind) {
case kAnyStore: dmb_flavor = kISH; break;
case kLoadAny: dmb_flavor = kISH; break;
// We conjecture that kISHLD is insufficient. It is documented
// to provide LoadLoad | StoreStore ordering. But if this were used
// to implement volatile loads, we suspect that the lack of store
// atomicity on ARM would cause us to allow incorrect results for
// the canonical IRIW example. But we're not sure.
// We should be using acquire loads instead.
case kStoreStore: dmb_flavor = kISHST; break;
case kAnyAny: dmb_flavor = kISH; break;
default:
LOG(FATAL) << "Unexpected MemBarrierKind: " << barrier_kind;
dmb_flavor = kSY; // quiet gcc.
break;
}
bool ret = false;
// If the same barrier already exists, don't generate another.
if (barrier == nullptr
|| (barrier->opcode != kA64Dmb1B || barrier->operands[0] != dmb_flavor)) {
barrier = NewLIR1(kA64Dmb1B, dmb_flavor);
ret = true;
}
// At this point we must have a memory barrier. Mark it as a scheduling barrier as well.
DCHECK(!barrier->flags.use_def_invalid);
barrier->u.m.def_mask = &kEncodeAll;
return ret;
}
void Arm64Mir2Lir::GenIntToLong(RegLocation rl_dest, RegLocation rl_src) {
RegLocation rl_result;
rl_src = LoadValue(rl_src, kCoreReg);
rl_result = EvalLocWide(rl_dest, kCoreReg, true);
NewLIR4(WIDE(kA64Sbfm4rrdd), rl_result.reg.GetReg(), As64BitReg(rl_src.reg).GetReg(), 0, 31);
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenDivRemLong(Instruction::Code opcode, RegLocation rl_dest,
RegLocation rl_src1, RegLocation rl_src2, bool is_div, int flags) {
if (rl_src2.is_const) {
DCHECK(rl_src2.wide);
int64_t lit = mir_graph_->ConstantValueWide(rl_src2);
if (HandleEasyDivRem64(opcode, is_div, rl_src1, rl_dest, lit)) {
return;
}
}
RegLocation rl_result;
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
rl_src2 = LoadValueWide(rl_src2, kCoreReg);
if ((flags & MIR_IGNORE_DIV_ZERO_CHECK) == 0) {
GenDivZeroCheck(rl_src2.reg);
}
rl_result = GenDivRem(rl_dest, rl_src1.reg, rl_src2.reg, is_div);
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenLongOp(OpKind op, RegLocation rl_dest, RegLocation rl_src1,
RegLocation rl_src2) {
RegLocation rl_result;
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
rl_src2 = LoadValueWide(rl_src2, kCoreReg);
rl_result = EvalLocWide(rl_dest, kCoreReg, true);
OpRegRegRegShift(op, rl_result.reg, rl_src1.reg, rl_src2.reg, ENCODE_NO_SHIFT);
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenNegLong(RegLocation rl_dest, RegLocation rl_src) {
RegLocation rl_result;
rl_src = LoadValueWide(rl_src, kCoreReg);
rl_result = EvalLocWide(rl_dest, kCoreReg, true);
OpRegRegShift(kOpNeg, rl_result.reg, rl_src.reg, ENCODE_NO_SHIFT);
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenNotLong(RegLocation rl_dest, RegLocation rl_src) {
RegLocation rl_result;
rl_src = LoadValueWide(rl_src, kCoreReg);
rl_result = EvalLocWide(rl_dest, kCoreReg, true);
OpRegRegShift(kOpMvn, rl_result.reg, rl_src.reg, ENCODE_NO_SHIFT);
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenArithOpLong(Instruction::Code opcode, RegLocation rl_dest,
RegLocation rl_src1, RegLocation rl_src2, int flags) {
switch (opcode) {
case Instruction::NOT_LONG:
GenNotLong(rl_dest, rl_src2);
return;
case Instruction::ADD_LONG:
case Instruction::ADD_LONG_2ADDR:
GenLongOp(kOpAdd, rl_dest, rl_src1, rl_src2);
return;
case Instruction::SUB_LONG:
case Instruction::SUB_LONG_2ADDR:
GenLongOp(kOpSub, rl_dest, rl_src1, rl_src2);
return;
case Instruction::MUL_LONG:
case Instruction::MUL_LONG_2ADDR:
GenLongOp(kOpMul, rl_dest, rl_src1, rl_src2);
return;
case Instruction::DIV_LONG:
case Instruction::DIV_LONG_2ADDR:
GenDivRemLong(opcode, rl_dest, rl_src1, rl_src2, /*is_div*/ true, flags);
return;
case Instruction::REM_LONG:
case Instruction::REM_LONG_2ADDR:
GenDivRemLong(opcode, rl_dest, rl_src1, rl_src2, /*is_div*/ false, flags);
return;
case Instruction::AND_LONG_2ADDR:
case Instruction::AND_LONG:
GenLongOp(kOpAnd, rl_dest, rl_src1, rl_src2);
return;
case Instruction::OR_LONG:
case Instruction::OR_LONG_2ADDR:
GenLongOp(kOpOr, rl_dest, rl_src1, rl_src2);
return;
case Instruction::XOR_LONG:
case Instruction::XOR_LONG_2ADDR:
GenLongOp(kOpXor, rl_dest, rl_src1, rl_src2);
return;
case Instruction::NEG_LONG: {
GenNegLong(rl_dest, rl_src2);
return;
}
default:
LOG(FATAL) << "Invalid long arith op";
return;
}
}
/*
* Generate array load
*/
void Arm64Mir2Lir::GenArrayGet(int opt_flags, OpSize size, RegLocation rl_array,
RegLocation rl_index, RegLocation rl_dest, int scale) {
RegisterClass reg_class = RegClassBySize(size);
int len_offset = mirror::Array::LengthOffset().Int32Value();
int data_offset;
RegLocation rl_result;
bool constant_index = rl_index.is_const;
rl_array = LoadValue(rl_array, kRefReg);
if (!constant_index) {
rl_index = LoadValue(rl_index, kCoreReg);
}
if (rl_dest.wide) {
data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Int32Value();
} else {
data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Int32Value();
}
/* null object? */
GenNullCheck(rl_array.reg, opt_flags);
bool needs_range_check = (!(opt_flags & MIR_IGNORE_RANGE_CHECK));
RegStorage reg_len;
if (needs_range_check) {
reg_len = AllocTemp();
/* Get len */
Load32Disp(rl_array.reg, len_offset, reg_len);
MarkPossibleNullPointerException(opt_flags);
} else {
ForceImplicitNullCheck(rl_array.reg, opt_flags);
}
if (constant_index) {
rl_result = EvalLoc(rl_dest, reg_class, true);
if (needs_range_check) {
GenArrayBoundsCheck(mir_graph_->ConstantValue(rl_index), reg_len);
FreeTemp(reg_len);
}
// Fold the constant index into the data offset.
data_offset += mir_graph_->ConstantValue(rl_index) << scale;
if (rl_result.ref) {
LoadRefDisp(rl_array.reg, data_offset, rl_result.reg, kNotVolatile);
} else {
LoadBaseDisp(rl_array.reg, data_offset, rl_result.reg, size, kNotVolatile);
}
} else {
// Offset base, then use indexed load.
RegStorage reg_ptr = AllocTempRef();
OpRegRegImm(kOpAdd, reg_ptr, rl_array.reg, data_offset);
FreeTemp(rl_array.reg);
rl_result = EvalLoc(rl_dest, reg_class, true);
if (needs_range_check) {
GenArrayBoundsCheck(rl_index.reg, reg_len);
FreeTemp(reg_len);
}
if (rl_result.ref) {
LoadRefIndexed(reg_ptr, rl_index.reg, rl_result.reg, scale);
} else {
LoadBaseIndexed(reg_ptr, rl_index.reg, rl_result.reg, scale, size);
}
FreeTemp(reg_ptr);
}
if (rl_dest.wide) {
StoreValueWide(rl_dest, rl_result);
} else {
StoreValue(rl_dest, rl_result);
}
}
/*
* Generate array store
*
*/
void Arm64Mir2Lir::GenArrayPut(int opt_flags, OpSize size, RegLocation rl_array,
RegLocation rl_index, RegLocation rl_src, int scale, bool card_mark) {
RegisterClass reg_class = RegClassBySize(size);
int len_offset = mirror::Array::LengthOffset().Int32Value();
bool constant_index = rl_index.is_const;
int data_offset;
if (size == k64 || size == kDouble) {
data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Int32Value();
} else {
data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Int32Value();
}
rl_array = LoadValue(rl_array, kRefReg);
if (!constant_index) {
rl_index = LoadValue(rl_index, kCoreReg);
}
RegStorage reg_ptr;
bool allocated_reg_ptr_temp = false;
if (constant_index) {
reg_ptr = rl_array.reg;
} else if (IsTemp(rl_array.reg) && !card_mark) {
Clobber(rl_array.reg);
reg_ptr = rl_array.reg;
} else {
allocated_reg_ptr_temp = true;
reg_ptr = AllocTempRef();
}
/* null object? */
GenNullCheck(rl_array.reg, opt_flags);
bool needs_range_check = (!(opt_flags & MIR_IGNORE_RANGE_CHECK));
RegStorage reg_len;
if (needs_range_check) {
reg_len = AllocTemp();
// NOTE: max live temps(4) here.
/* Get len */
Load32Disp(rl_array.reg, len_offset, reg_len);
MarkPossibleNullPointerException(opt_flags);
} else {
ForceImplicitNullCheck(rl_array.reg, opt_flags);
}
/* at this point, reg_ptr points to array, 2 live temps */
if (rl_src.wide) {
rl_src = LoadValueWide(rl_src, reg_class);
} else {
rl_src = LoadValue(rl_src, reg_class);
}
if (constant_index) {
if (needs_range_check) {
GenArrayBoundsCheck(mir_graph_->ConstantValue(rl_index), reg_len);
FreeTemp(reg_len);
}
// Fold the constant index into the data offset.
data_offset += mir_graph_->ConstantValue(rl_index) << scale;
if (rl_src.ref) {
StoreRefDisp(reg_ptr, data_offset, rl_src.reg, kNotVolatile);
} else {
StoreBaseDisp(reg_ptr, data_offset, rl_src.reg, size, kNotVolatile);
}
} else {
/* reg_ptr -> array data */
OpRegRegImm(kOpAdd, reg_ptr, rl_array.reg, data_offset);
if (needs_range_check) {
GenArrayBoundsCheck(rl_index.reg, reg_len);
FreeTemp(reg_len);
}
if (rl_src.ref) {
StoreRefIndexed(reg_ptr, rl_index.reg, rl_src.reg, scale);
} else {
StoreBaseIndexed(reg_ptr, rl_index.reg, rl_src.reg, scale, size);
}
}
if (allocated_reg_ptr_temp) {
FreeTemp(reg_ptr);
}
if (card_mark) {
MarkGCCard(opt_flags, rl_src.reg, rl_array.reg);
}
}
void Arm64Mir2Lir::GenShiftImmOpLong(Instruction::Code opcode,
RegLocation rl_dest, RegLocation rl_src, RegLocation rl_shift,
int flags ATTRIBUTE_UNUSED) {
OpKind op = kOpBkpt;
// Per spec, we only care about low 6 bits of shift amount.
int shift_amount = mir_graph_->ConstantValue(rl_shift) & 0x3f;
rl_src = LoadValueWide(rl_src, kCoreReg);
if (shift_amount == 0) {
StoreValueWide(rl_dest, rl_src);
return;
}
RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true);
switch (opcode) {
case Instruction::SHL_LONG:
case Instruction::SHL_LONG_2ADDR:
op = kOpLsl;
break;
case Instruction::SHR_LONG:
case Instruction::SHR_LONG_2ADDR:
op = kOpAsr;
break;
case Instruction::USHR_LONG:
case Instruction::USHR_LONG_2ADDR:
op = kOpLsr;
break;
default:
LOG(FATAL) << "Unexpected case";
}
OpRegRegImm(op, rl_result.reg, rl_src.reg, shift_amount);
StoreValueWide(rl_dest, rl_result);
}
void Arm64Mir2Lir::GenArithImmOpLong(Instruction::Code opcode, RegLocation rl_dest,
RegLocation rl_src1, RegLocation rl_src2, int flags) {
OpKind op = kOpBkpt;
switch (opcode) {
case Instruction::ADD_LONG:
case Instruction::ADD_LONG_2ADDR:
op = kOpAdd;
break;
case Instruction::SUB_LONG:
case Instruction::SUB_LONG_2ADDR:
op = kOpSub;
break;
case Instruction::AND_LONG:
case Instruction::AND_LONG_2ADDR:
op = kOpAnd;
break;
case Instruction::OR_LONG:
case Instruction::OR_LONG_2ADDR:
op = kOpOr;
break;
case Instruction::XOR_LONG:
case Instruction::XOR_LONG_2ADDR:
op = kOpXor;
break;
default:
LOG(FATAL) << "Unexpected opcode";
}
if (op == kOpSub) {
if (!rl_src2.is_const) {
return GenArithOpLong(opcode, rl_dest, rl_src1, rl_src2, flags);
}
} else {
// Associativity.
if (!rl_src2.is_const) {
DCHECK(rl_src1.is_const);
std::swap(rl_src1, rl_src2);
}
}
DCHECK(rl_src2.is_const);
int64_t val = mir_graph_->ConstantValueWide(rl_src2);
rl_src1 = LoadValueWide(rl_src1, kCoreReg);
RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true);
OpRegRegImm64(op, rl_result.reg, rl_src1.reg, val);
StoreValueWide(rl_dest, rl_result);
}
static uint32_t ExtractReg(uint32_t reg_mask, int* reg) {
// Find first register.
int first_bit_set = CTZ(reg_mask) + 1;
*reg = *reg + first_bit_set;
reg_mask >>= first_bit_set;
return reg_mask;
}
/**
* @brief Split a register list in pairs or registers.
*
* Given a list of registers in @p reg_mask, split the list in pairs. Use as follows:
* @code
* int reg1 = -1, reg2 = -1;
* while (reg_mask) {
* reg_mask = GenPairWise(reg_mask, & reg1, & reg2);
* if (UNLIKELY(reg2 < 0)) {
* // Single register in reg1.
* } else {
* // Pair in reg1, reg2.
* }
* }
* @endcode
*/
static uint32_t GenPairWise(uint32_t reg_mask, int* reg1, int* reg2) {
// Find first register.
int first_bit_set = CTZ(reg_mask) + 1;
int reg = *reg1 + first_bit_set;
reg_mask >>= first_bit_set;
if (LIKELY(reg_mask)) {
// Save the first register, find the second and use the pair opcode.
int second_bit_set = CTZ(reg_mask) + 1;
*reg2 = reg;
reg_mask >>= second_bit_set;
*reg1 = reg + second_bit_set;
return reg_mask;
}
// Use the single opcode, as we just have one register.
*reg1 = reg;
*reg2 = -1;
return reg_mask;
}
static dwarf::Reg DwarfCoreReg(int num) {
return dwarf::Reg::Arm64Core(num);
}
static dwarf::Reg DwarfFpReg(int num) {
return dwarf::Reg::Arm64Fp(num);
}
static void SpillCoreRegs(Arm64Mir2Lir* m2l, RegStorage base, int offset, uint32_t reg_mask) {
int reg1 = -1, reg2 = -1;
const int reg_log2_size = 3;
for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) {
reg_mask = GenPairWise(reg_mask, & reg1, & reg2);
if (UNLIKELY(reg2 < 0)) {
m2l->NewLIR3(WIDE(kA64Str3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset);
m2l->cfi().RelOffset(DwarfCoreReg(reg1), offset << reg_log2_size);
} else {
m2l->NewLIR4(WIDE(kA64Stp4rrXD), RegStorage::Solo64(reg2).GetReg(),
RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset);
m2l->cfi().RelOffset(DwarfCoreReg(reg2), offset << reg_log2_size);
m2l->cfi().RelOffset(DwarfCoreReg(reg1), (offset + 1) << reg_log2_size);
}
}
}
// TODO(Arm64): consider using ld1 and st1?
static void SpillFPRegs(Arm64Mir2Lir* m2l, RegStorage base, int offset, uint32_t reg_mask) {
int reg1 = -1, reg2 = -1;
const int reg_log2_size = 3;
for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) {
reg_mask = GenPairWise(reg_mask, & reg1, & reg2);
if (UNLIKELY(reg2 < 0)) {
m2l->NewLIR3(WIDE(kA64Str3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(),
offset);
m2l->cfi().RelOffset(DwarfFpReg(reg1), offset << reg_log2_size);
} else {
m2l->NewLIR4(WIDE(kA64Stp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(),
RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset);
m2l->cfi().RelOffset(DwarfFpReg(reg2), offset << reg_log2_size);
m2l->cfi().RelOffset(DwarfFpReg(reg1), (offset + 1) << reg_log2_size);
}
}
}
static int SpillRegsPreSub(Arm64Mir2Lir* m2l, uint32_t core_reg_mask, uint32_t fp_reg_mask,
int frame_size) {
m2l->OpRegRegImm(kOpSub, rs_sp, rs_sp, frame_size);
m2l->cfi().AdjustCFAOffset(frame_size);
int core_count = POPCOUNT(core_reg_mask);
if (fp_reg_mask != 0) {
// Spill FP regs.
int fp_count = POPCOUNT(fp_reg_mask);
int spill_offset = frame_size - (core_count + fp_count) * kArm64PointerSize;
SpillFPRegs(m2l, rs_sp, spill_offset, fp_reg_mask);
}
if (core_reg_mask != 0) {
// Spill core regs.
int spill_offset = frame_size - (core_count * kArm64PointerSize);
SpillCoreRegs(m2l, rs_sp, spill_offset, core_reg_mask);
}
return frame_size;
}
static int SpillRegsPreIndexed(Arm64Mir2Lir* m2l, RegStorage base, uint32_t core_reg_mask,
uint32_t fp_reg_mask) {
// Otherwise, spill both core and fp regs at the same time.
// The very first instruction will be an stp with pre-indexed address, moving the stack pointer
// down. From then on, we fill upwards. This will generate overall the same number of instructions
// as the specialized code above in most cases (exception being odd number of core and even
// non-zero fp spills), but is more flexible, as the offsets are guaranteed small.
//
// Some demonstrative fill cases : (c) = core, (f) = fp
// cc 44 cc 44 cc 22 cc 33 fc => 1[1/2]
// fc => 23 fc => 23 ff => 11 ff => 22
// ff 11 f 11 f 11
//
int reg1 = -1, reg2 = -1;
int core_count = POPCOUNT(core_reg_mask);
int fp_count = POPCOUNT(fp_reg_mask);
int combined = fp_count + core_count;
int all_offset = RoundUp(combined, 2); // Needs to be 16B = 2-reg aligned.
int cur_offset = 2; // What's the starting offset after the first stp? We expect the base slot
// to be filled.
// First figure out whether the bottom is FP or core.
if (fp_count > 0) {
// Some FP spills.
//
// Four cases: (d0 is dummy to fill up stp)
// 1) Single FP, even number of core -> stp d0, fp_reg
// 2) Single FP, odd number of core -> stp fp_reg, d0
// 3) More FP, even number combined -> stp fp_reg1, fp_reg2
// 4) More FP, odd number combined -> stp d0, fp_reg
if (fp_count == 1) {
fp_reg_mask = ExtractReg(fp_reg_mask, ®1);
DCHECK_EQ(fp_reg_mask, 0U);
if (core_count % 2 == 0) {
m2l->NewLIR4(WIDE(kA64StpPre4ffXD),
RegStorage::FloatSolo64(reg1).GetReg(),
RegStorage::FloatSolo64(reg1).GetReg(),
base.GetReg(), -all_offset);
m2l->cfi().AdjustCFAOffset(all_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfFpReg(reg1), kArm64PointerSize);
} else {
m2l->NewLIR4(WIDE(kA64StpPre4ffXD),
RegStorage::FloatSolo64(reg1).GetReg(),
RegStorage::FloatSolo64(reg1).GetReg(),
base.GetReg(), -all_offset);
m2l->cfi().AdjustCFAOffset(all_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfFpReg(reg1), 0);
cur_offset = 0; // That core reg needs to go into the upper half.
}
} else {
if (combined % 2 == 0) {
fp_reg_mask = GenPairWise(fp_reg_mask, ®1, ®2);
m2l->NewLIR4(WIDE(kA64StpPre4ffXD), RegStorage::FloatSolo64(reg2).GetReg(),
RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), -all_offset);
m2l->cfi().AdjustCFAOffset(all_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfFpReg(reg2), 0);
m2l->cfi().RelOffset(DwarfFpReg(reg1), kArm64PointerSize);
} else {
fp_reg_mask = ExtractReg(fp_reg_mask, ®1);
m2l->NewLIR4(WIDE(kA64StpPre4ffXD), rs_d0.GetReg(), RegStorage::FloatSolo64(reg1).GetReg(),
base.GetReg(), -all_offset);
m2l->cfi().AdjustCFAOffset(all_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfFpReg(reg1), kArm64PointerSize);
}
}
} else {
// No FP spills.
//
// Two cases:
// 1) Even number of core -> stp core1, core2
// 2) Odd number of core -> stp xzr, core1
if (core_count % 2 == 1) {
core_reg_mask = ExtractReg(core_reg_mask, ®1);
m2l->NewLIR4(WIDE(kA64StpPre4rrXD), rs_xzr.GetReg(),
RegStorage::Solo64(reg1).GetReg(), base.GetReg(), -all_offset);
m2l->cfi().AdjustCFAOffset(all_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfCoreReg(reg1), kArm64PointerSize);
} else {
core_reg_mask = GenPairWise(core_reg_mask, ®1, ®2);
m2l->NewLIR4(WIDE(kA64StpPre4rrXD), RegStorage::Solo64(reg2).GetReg(),
RegStorage::Solo64(reg1).GetReg(), base.GetReg(), -all_offset);
m2l->cfi().AdjustCFAOffset(all_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfCoreReg(reg2), 0);
m2l->cfi().RelOffset(DwarfCoreReg(reg1), kArm64PointerSize);
}
}
DCHECK_EQ(m2l->cfi().GetCurrentCFAOffset(),
static_cast<int>(all_offset * kArm64PointerSize));
if (fp_count != 0) {
for (; fp_reg_mask != 0;) {
// Have some FP regs to do.
fp_reg_mask = GenPairWise(fp_reg_mask, ®1, ®2);
if (UNLIKELY(reg2 < 0)) {
m2l->NewLIR3(WIDE(kA64Str3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(),
cur_offset);
m2l->cfi().RelOffset(DwarfFpReg(reg1), cur_offset * kArm64PointerSize);
// Do not increment offset here, as the second half will be filled by a core reg.
} else {
m2l->NewLIR4(WIDE(kA64Stp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(),
RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), cur_offset);
m2l->cfi().RelOffset(DwarfFpReg(reg2), cur_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfFpReg(reg1), (cur_offset + 1) * kArm64PointerSize);
cur_offset += 2;
}
}
// Reset counting.
reg1 = -1;
// If there is an odd number of core registers, we need to store the bottom now.
if (core_count % 2 == 1) {
core_reg_mask = ExtractReg(core_reg_mask, ®1);
m2l->NewLIR3(WIDE(kA64Str3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(),
cur_offset + 1);
m2l->cfi().RelOffset(DwarfCoreReg(reg1), (cur_offset + 1) * kArm64PointerSize);
cur_offset += 2; // Half-slot filled now.
}
}
// Spill the rest of the core regs. They are guaranteed to be even.
DCHECK_EQ(POPCOUNT(core_reg_mask) % 2, 0);
for (; core_reg_mask != 0; cur_offset += 2) {
core_reg_mask = GenPairWise(core_reg_mask, ®1, ®2);
m2l->NewLIR4(WIDE(kA64Stp4rrXD), RegStorage::Solo64(reg2).GetReg(),
RegStorage::Solo64(reg1).GetReg(), base.GetReg(), cur_offset);
m2l->cfi().RelOffset(DwarfCoreReg(reg2), cur_offset * kArm64PointerSize);
m2l->cfi().RelOffset(DwarfCoreReg(reg1), (cur_offset + 1) * kArm64PointerSize);
}
DCHECK_EQ(cur_offset, all_offset);
return all_offset * 8;
}
int Arm64Mir2Lir::SpillRegs(RegStorage base, uint32_t core_reg_mask, uint32_t fp_reg_mask,
int frame_size) {
// If the frame size is small enough that all offsets would fit into the immediates, use that
// setup, as it decrements sp early (kind of instruction scheduling), and is not worse
// instruction-count wise than the complicated code below.
//
// This case is also optimal when we have an odd number of core spills, and an even (non-zero)
// number of fp spills.
if ((RoundUp(frame_size, 8) / 8 <= 63)) {
return SpillRegsPreSub(this, core_reg_mask, fp_reg_mask, frame_size);
} else {
return SpillRegsPreIndexed(this, base, core_reg_mask, fp_reg_mask);
}
}
static void UnSpillCoreRegs(Arm64Mir2Lir* m2l, RegStorage base, int offset, uint32_t reg_mask) {
int reg1 = -1, reg2 = -1;
const int reg_log2_size = 3;
for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) {
reg_mask = GenPairWise(reg_mask, & reg1, & reg2);
if (UNLIKELY(reg2 < 0)) {
m2l->NewLIR3(WIDE(kA64Ldr3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset);
m2l->cfi().Restore(DwarfCoreReg(reg1));
} else {
DCHECK_LE(offset, 63);
m2l->NewLIR4(WIDE(kA64Ldp4rrXD), RegStorage::Solo64(reg2).GetReg(),
RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset);
m2l->cfi().Restore(DwarfCoreReg(reg2));
m2l->cfi().Restore(DwarfCoreReg(reg1));
}
}
}
static void UnSpillFPRegs(Arm64Mir2Lir* m2l, RegStorage base, int offset, uint32_t reg_mask) {
int reg1 = -1, reg2 = -1;
const int reg_log2_size = 3;
for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) {
reg_mask = GenPairWise(reg_mask, & reg1, & reg2);
if (UNLIKELY(reg2 < 0)) {
m2l->NewLIR3(WIDE(kA64Ldr3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(),
offset);
m2l->cfi().Restore(DwarfFpReg(reg1));
} else {
m2l->NewLIR4(WIDE(kA64Ldp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(),
RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset);
m2l->cfi().Restore(DwarfFpReg(reg2));
m2l->cfi().Restore(DwarfFpReg(reg1));
}
}
}
void Arm64Mir2Lir::UnspillRegs(RegStorage base, uint32_t core_reg_mask, uint32_t fp_reg_mask,
int frame_size) {
DCHECK_EQ(base, rs_sp);
// Restore saves and drop stack frame.
// 2 versions:
//
// 1. (Original): Try to address directly, then drop the whole frame.
// Limitation: ldp is a 7b signed immediate.
//
// 2. (New): Drop the non-save-part. Then do similar to original, which is now guaranteed to be
// in range. Then drop the rest.
//
// TODO: In methods with few spills but huge frame, it would be better to do non-immediate loads
// in variant 1.
// "Magic" constant, 63 (max signed 7b) * 8.
static constexpr int kMaxFramesizeForOffset = 63 * kArm64PointerSize;
const int num_core_spills = POPCOUNT(core_reg_mask);
const int num_fp_spills = POPCOUNT(fp_reg_mask);
int early_drop = 0;
if (frame_size > kMaxFramesizeForOffset) {
// Second variant. Drop the frame part.
// TODO: Always use the first formula, as num_fp_spills would be zero?
if (fp_reg_mask != 0) {
early_drop = frame_size - kArm64PointerSize * (num_fp_spills + num_core_spills);
} else {
early_drop = frame_size - kArm64PointerSize * num_core_spills;
}
// Drop needs to be 16B aligned, so that SP keeps aligned.
early_drop = RoundDown(early_drop, 16);
OpRegImm64(kOpAdd, rs_sp, early_drop);
cfi_.AdjustCFAOffset(-early_drop);
}
// Unspill.
if (fp_reg_mask != 0) {
int offset = frame_size - early_drop - kArm64PointerSize * (num_fp_spills + num_core_spills);
UnSpillFPRegs(this, rs_sp, offset, fp_reg_mask);
}
if (core_reg_mask != 0) {
int offset = frame_size - early_drop - kArm64PointerSize * num_core_spills;
UnSpillCoreRegs(this, rs_sp, offset, core_reg_mask);
}
// Drop the (rest of) the frame.
int adjust = frame_size - early_drop;
OpRegImm64(kOpAdd, rs_sp, adjust);
cfi_.AdjustCFAOffset(-adjust);
}
bool Arm64Mir2Lir::GenInlinedReverseBits(CallInfo* info, OpSize size) {
A64Opcode wide = IsWide(size) ? WIDE(0) : UNWIDE(0);
RegLocation rl_src_i = info->args[0];
RegLocation rl_dest = IsWide(size) ? InlineTargetWide(info) : InlineTarget(info); // result reg
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
RegLocation rl_i = IsWide(size) ?
LoadValueWide(rl_src_i, kCoreReg) : LoadValue(rl_src_i, kCoreReg);
NewLIR2(kA64Rbit2rr | wide, rl_result.reg.GetReg(), rl_i.reg.GetReg());
IsWide(size) ? StoreValueWide(rl_dest, rl_result) : StoreValue(rl_dest, rl_result);
return true;
}
} // namespace art