/*
* Copyright (C) 2011 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "DFGJITCompiler.h"
#if ENABLE(DFG_JIT)
#include "CodeBlock.h"
#include "DFGJITCodeGenerator.h"
#include "DFGNonSpeculativeJIT.h"
#include "DFGOperations.h"
#include "DFGRegisterBank.h"
#include "DFGSpeculativeJIT.h"
#include "JSGlobalData.h"
#include "LinkBuffer.h"
namespace JSC { namespace DFG {
// This method used to fill a numeric value to a FPR when linking speculative -> non-speculative.
void JITCompiler::fillNumericToDouble(NodeIndex nodeIndex, FPRReg fpr, GPRReg temporary)
{
Node& node = graph()[nodeIndex];
MacroAssembler::RegisterID tempReg = gprToRegisterID(temporary);
if (node.isConstant()) {
ASSERT(node.op == DoubleConstant);
move(MacroAssembler::ImmPtr(reinterpret_cast<void*>(reinterpretDoubleToIntptr(valueOfDoubleConstant(nodeIndex)))), tempReg);
movePtrToDouble(tempReg, fprToRegisterID(fpr));
} else {
loadPtr(addressFor(node.virtualRegister), tempReg);
Jump isInteger = branchPtr(MacroAssembler::AboveOrEqual, tempReg, tagTypeNumberRegister);
jitAssertIsJSDouble(gpr0);
addPtr(tagTypeNumberRegister, tempReg);
movePtrToDouble(tempReg, fprToRegisterID(fpr));
Jump hasUnboxedDouble = jump();
isInteger.link(this);
convertInt32ToDouble(tempReg, fprToRegisterID(fpr));
hasUnboxedDouble.link(this);
}
}
// This method used to fill an integer value to a GPR when linking speculative -> non-speculative.
void JITCompiler::fillInt32ToInteger(NodeIndex nodeIndex, GPRReg gpr)
{
Node& node = graph()[nodeIndex];
if (node.isConstant()) {
ASSERT(node.op == Int32Constant);
move(MacroAssembler::Imm32(valueOfInt32Constant(nodeIndex)), gprToRegisterID(gpr));
} else {
#if DFG_JIT_ASSERT
// Redundant load, just so we can check the tag!
loadPtr(addressFor(node.virtualRegister), gprToRegisterID(gpr));
jitAssertIsJSInt32(gpr);
#endif
load32(addressFor(node.virtualRegister), gprToRegisterID(gpr));
}
}
// This method used to fill a JSValue to a GPR when linking speculative -> non-speculative.
void JITCompiler::fillToJS(NodeIndex nodeIndex, GPRReg gpr)
{
Node& node = graph()[nodeIndex];
if (node.isConstant()) {
if (isInt32Constant(nodeIndex)) {
JSValue jsValue = jsNumber(valueOfInt32Constant(nodeIndex));
move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gprToRegisterID(gpr));
} else if (isDoubleConstant(nodeIndex)) {
JSValue jsValue(JSValue::EncodeAsDouble, valueOfDoubleConstant(nodeIndex));
move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gprToRegisterID(gpr));
} else {
ASSERT(isJSConstant(nodeIndex));
JSValue jsValue = valueOfJSConstant(nodeIndex);
move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gprToRegisterID(gpr));
}
return;
}
loadPtr(addressFor(node.virtualRegister), gprToRegisterID(gpr));
}
void JITCompiler::jumpFromSpeculativeToNonSpeculative(const SpeculationCheck& check, const EntryLocation& entry, SpeculationRecovery* recovery)
{
ASSERT(check.m_nodeIndex == entry.m_nodeIndex);
// Link the jump from the Speculative path to here.
check.m_check.link(this);
// Does this speculation check require any additional recovery to be performed,
// to restore any state that has been overwritten before we enter back in to the
// non-speculative path.
if (recovery) {
// The only additional recovery we currently support is for integer add operation
ASSERT(recovery->type() == SpeculativeAdd);
// Revert the add.
sub32(gprToRegisterID(recovery->src()), gprToRegisterID(recovery->dest()));
}
// FIXME: - This is hideously inefficient!
// Where a value is live in a register in the speculative path, and is required in a register
// on the non-speculative path, we should not need to be spilling it and reloading (we may
// need to spill anyway, if the value is marked as spilled on the non-speculative path).
// This may also be spilling values that don't need spilling, e.g. are already spilled,
// are constants, or are arguments.
// Spill all GPRs in use by the speculative path.
for (GPRReg gpr = gpr0; gpr < numberOfGPRs; next(gpr)) {
NodeIndex nodeIndex = check.m_gprInfo[gpr].nodeIndex;
if (nodeIndex == NoNode)
continue;
DataFormat dataFormat = check.m_gprInfo[gpr].format;
VirtualRegister virtualRegister = graph()[nodeIndex].virtualRegister;
ASSERT(dataFormat == DataFormatInteger || DataFormatCell || dataFormat & DataFormatJS);
if (dataFormat == DataFormatInteger)
orPtr(tagTypeNumberRegister, gprToRegisterID(gpr));
storePtr(gprToRegisterID(gpr), addressFor(virtualRegister));
}
// Spill all FPRs in use by the speculative path.
for (FPRReg fpr = fpr0; fpr < numberOfFPRs; next(fpr)) {
NodeIndex nodeIndex = check.m_fprInfo[fpr];
if (nodeIndex == NoNode)
continue;
VirtualRegister virtualRegister = graph()[nodeIndex].virtualRegister;
moveDoubleToPtr(fprToRegisterID(fpr), regT0);
subPtr(tagTypeNumberRegister, regT0);
storePtr(regT0, addressFor(virtualRegister));
}
// Fill all FPRs in use by the non-speculative path.
for (FPRReg fpr = fpr0; fpr < numberOfFPRs; next(fpr)) {
NodeIndex nodeIndex = entry.m_fprInfo[fpr];
if (nodeIndex == NoNode)
continue;
fillNumericToDouble(nodeIndex, fpr, gpr0);
}
// Fill all GPRs in use by the non-speculative path.
for (GPRReg gpr = gpr0; gpr < numberOfGPRs; next(gpr)) {
NodeIndex nodeIndex = entry.m_gprInfo[gpr].nodeIndex;
if (nodeIndex == NoNode)
continue;
DataFormat dataFormat = entry.m_gprInfo[gpr].format;
if (dataFormat == DataFormatInteger)
fillInt32ToInteger(nodeIndex, gpr);
else {
ASSERT(dataFormat & DataFormatJS || dataFormat == DataFormatCell); // Treat cell as JSValue for now!
fillToJS(nodeIndex, gpr);
// FIXME: For subtypes of DataFormatJS, should jitAssert the subtype?
}
}
// Jump into the non-speculative path.
jump(entry.m_entry);
}
void JITCompiler::linkSpeculationChecks(SpeculativeJIT& speculative, NonSpeculativeJIT& nonSpeculative)
{
// Iterators to walk over the set of bail outs & corresponding entry points.
SpeculationCheckVector::Iterator checksIter = speculative.speculationChecks().begin();
SpeculationCheckVector::Iterator checksEnd = speculative.speculationChecks().end();
NonSpeculativeJIT::EntryLocationVector::Iterator entriesIter = nonSpeculative.entryLocations().begin();
NonSpeculativeJIT::EntryLocationVector::Iterator entriesEnd = nonSpeculative.entryLocations().end();
// Iterate over the speculation checks.
while (checksIter != checksEnd) {
// For every bail out from the speculative path, we must have provided an entry point
// into the non-speculative one.
ASSERT(checksIter->m_nodeIndex == entriesIter->m_nodeIndex);
// There may be multiple bail outs that map to the same entry point!
do {
ASSERT(checksIter != checksEnd);
ASSERT(entriesIter != entriesEnd);
// Plant code to link this speculation failure.
const SpeculationCheck& check = *checksIter;
const EntryLocation& entry = *entriesIter;
jumpFromSpeculativeToNonSpeculative(check, entry, speculative.speculationRecovery(check.m_recoveryIndex));
++checksIter;
} while (checksIter != checksEnd && checksIter->m_nodeIndex == entriesIter->m_nodeIndex);
++entriesIter;
}
// FIXME: https://bugs.webkit.org/show_bug.cgi?id=56289
ASSERT(!(checksIter != checksEnd));
ASSERT(!(entriesIter != entriesEnd));
}
void JITCompiler::compileFunction(JITCode& entry, MacroAssemblerCodePtr& entryWithArityCheck)
{
// === Stage 1 - Function header code generation ===
//
// This code currently matches the old JIT. In the function header we need to
// pop the return address (since we do not allow any recursion on the machine
// stack), and perform a fast register file check.
// This is the main entry point, without performing an arity check.
// FIXME: https://bugs.webkit.org/show_bug.cgi?id=56292
// We'll need to convert the remaining cti_ style calls (specifically the register file
// check) which will be dependent on stack layout. (We'd need to account for this in
// both normal return code and when jumping to an exception handler).
preserveReturnAddressAfterCall(regT2);
emitPutToCallFrameHeader(regT2, RegisterFile::ReturnPC);
// If we needed to perform an arity check we will already have moved the return address,
// so enter after this.
Label fromArityCheck(this);
// Setup a pointer to the codeblock in the CallFrameHeader.
emitPutImmediateToCallFrameHeader(m_codeBlock, RegisterFile::CodeBlock);
// Plant a check that sufficient space is available in the RegisterFile.
// FIXME: https://bugs.webkit.org/show_bug.cgi?id=56291
addPtr(Imm32(m_codeBlock->m_numCalleeRegisters * sizeof(Register)), callFrameRegister, regT1);
Jump registerFileCheck = branchPtr(Below, AbsoluteAddress(m_globalData->interpreter->registerFile().addressOfEnd()), regT1);
// Return here after register file check.
Label fromRegisterFileCheck = label();
// === Stage 2 - Function body code generation ===
//
// We generate the speculative code path, followed by the non-speculative
// code for the function. Next we need to link the two together, making
// bail-outs from the speculative path jump to the corresponding point on
// the non-speculative one (and generating any code necessary to juggle
// register values around, rebox values, and ensure spilled, to match the
// non-speculative path's requirements).
#if DFG_JIT_BREAK_ON_EVERY_FUNCTION
// Handy debug tool!
breakpoint();
#endif
// First generate the speculative path.
Label speculativePathBegin = label();
SpeculativeJIT speculative(*this);
bool compiledSpeculative = speculative.compile();
// Next, generate the non-speculative path. We pass this a SpeculationCheckIndexIterator
// to allow it to check which nodes in the graph may bail out, and may need to reenter the
// non-speculative path.
if (compiledSpeculative) {
SpeculationCheckIndexIterator checkIterator(speculative.speculationChecks());
NonSpeculativeJIT nonSpeculative(*this);
nonSpeculative.compile(checkIterator);
// Link the bail-outs from the speculative path to the corresponding entry points into the non-speculative one.
linkSpeculationChecks(speculative, nonSpeculative);
} else {
// If compilation through the SpeculativeJIT failed, throw away the code we generated.
m_calls.clear();
rewindToLabel(speculativePathBegin);
SpeculationCheckVector noChecks;
SpeculationCheckIndexIterator checkIterator(noChecks);
NonSpeculativeJIT nonSpeculative(*this);
nonSpeculative.compile(checkIterator);
}
// === Stage 3 - Function footer code generation ===
//
// Generate code to lookup and jump to exception handlers, to perform the slow
// register file check (if the fast one in the function header fails), and
// generate the entry point with arity check.
// Iterate over the m_calls vector, checking for exception checks,
// and linking them to here.
unsigned exceptionCheckCount = 0;
for (unsigned i = 0; i < m_calls.size(); ++i) {
Jump& exceptionCheck = m_calls[i].m_exceptionCheck;
if (exceptionCheck.isSet()) {
exceptionCheck.link(this);
++exceptionCheckCount;
}
}
// If any exception checks were linked, generate code to lookup a handler.
if (exceptionCheckCount) {
// lookupExceptionHandler is passed two arguments, exec (the CallFrame*), and
// an identifier for the operation that threw the exception, which we can use
// to look up handler information. The identifier we use is the return address
// of the call out from JIT code that threw the exception; this is still
// available on the stack, just below the stack pointer!
move(callFrameRegister, argumentRegister0);
peek(argumentRegister1, -1);
m_calls.append(CallRecord(call(), lookupExceptionHandler));
// lookupExceptionHandler leaves the handler CallFrame* in the returnValueRegister,
// and the address of the handler in returnValueRegister2.
jump(returnValueRegister2);
}
// Generate the register file check; if the fast check in the function head fails,
// we need to call out to a helper function to check whether more space is available.
// FIXME: change this from a cti call to a DFG style operation (normal C calling conventions).
registerFileCheck.link(this);
move(stackPointerRegister, argumentRegister0);
poke(callFrameRegister, OBJECT_OFFSETOF(struct JITStackFrame, callFrame) / sizeof(void*));
Call callRegisterFileCheck = call();
jump(fromRegisterFileCheck);
// The fast entry point into a function does not check the correct number of arguments
// have been passed to the call (we only use the fast entry point where we can statically
// determine the correct number of arguments have been passed, or have already checked).
// In cases where an arity check is necessary, we enter here.
// FIXME: change this from a cti call to a DFG style operation (normal C calling conventions).
Label arityCheck = label();
preserveReturnAddressAfterCall(regT2);
emitPutToCallFrameHeader(regT2, RegisterFile::ReturnPC);
branch32(Equal, regT1, Imm32(m_codeBlock->m_numParameters)).linkTo(fromArityCheck, this);
move(stackPointerRegister, argumentRegister0);
poke(callFrameRegister, OBJECT_OFFSETOF(struct JITStackFrame, callFrame) / sizeof(void*));
Call callArityCheck = call();
move(regT0, callFrameRegister);
jump(fromArityCheck);
// === Stage 4 - Link ===
//
// Link the code, populate data in CodeBlock data structures.
LinkBuffer linkBuffer(this, m_globalData->executableAllocator.poolForSize(m_assembler.size()), 0);
#if DFG_DEBUG_VERBOSE
fprintf(stderr, "JIT code start at %p\n", linkBuffer.debugAddress());
#endif
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
if (m_codeBlock->needsCallReturnIndices()) {
m_codeBlock->callReturnIndexVector().reserveCapacity(exceptionCheckCount);
for (unsigned i = 0; i < m_calls.size(); ++i) {
if (m_calls[i].m_exceptionCheck.isSet()) {
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_calls[i].m_call);
unsigned exceptionInfo = m_calls[i].m_exceptionInfo;
m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo));
}
}
}
// FIXME: switch the register file check & arity check over to DFGOpertaion style calls, not JIT stubs.
linkBuffer.link(callRegisterFileCheck, cti_register_file_check);
linkBuffer.link(callArityCheck, m_codeBlock->m_isConstructor ? cti_op_construct_arityCheck : cti_op_call_arityCheck);
entryWithArityCheck = linkBuffer.locationOf(arityCheck);
entry = linkBuffer.finalizeCode();
}
#if DFG_JIT_ASSERT
void JITCompiler::jitAssertIsInt32(GPRReg gpr)
{
#if CPU(X86_64)
Jump checkInt32 = branchPtr(BelowOrEqual, gprToRegisterID(gpr), TrustedImmPtr(reinterpret_cast<void*>(static_cast<uintptr_t>(0xFFFFFFFFu))));
breakpoint();
checkInt32.link(this);
#else
UNUSED_PARAM(gpr);
#endif
}
void JITCompiler::jitAssertIsJSInt32(GPRReg gpr)
{
Jump checkJSInt32 = branchPtr(AboveOrEqual, gprToRegisterID(gpr), tagTypeNumberRegister);
breakpoint();
checkJSInt32.link(this);
}
void JITCompiler::jitAssertIsJSNumber(GPRReg gpr)
{
Jump checkJSNumber = branchTestPtr(MacroAssembler::NonZero, gprToRegisterID(gpr), tagTypeNumberRegister);
breakpoint();
checkJSNumber.link(this);
}
void JITCompiler::jitAssertIsJSDouble(GPRReg gpr)
{
Jump checkJSInt32 = branchPtr(AboveOrEqual, gprToRegisterID(gpr), tagTypeNumberRegister);
Jump checkJSNumber = branchTestPtr(MacroAssembler::NonZero, gprToRegisterID(gpr), tagTypeNumberRegister);
checkJSInt32.link(this);
breakpoint();
checkJSNumber.link(this);
}
#endif
#if ENABLE(SAMPLING_COUNTERS) && CPU(X86_64) // Or any other 64-bit platform!
void JITCompiler::emitCount(AbstractSamplingCounter& counter, uint32_t increment)
{
addPtr(TrustedImm32(increment), AbsoluteAddress(counter.addressOfCounter()));
}
#endif
#if ENABLE(SAMPLING_COUNTERS) && CPU(X86) // Or any other little-endian 32-bit platform!
void JITCompiler::emitCount(AbstractSamplingCounter& counter, uint32_t increment)
{
intptr_t hiWord = reinterpret_cast<intptr_t>(counter.addressOfCounter()) + sizeof(int32_t);
add32(TrustedImm32(increment), AbsoluteAddress(counter.addressOfCounter()));
addWithCarry32(TrustedImm32(0), AbsoluteAddress(reinterpret_cast<void*>(hiWord)));
}
#endif
} } // namespace JSC::DFG
#endif