//===-- AMDILPeepholeOptimizer.cpp - AMDIL Peephole optimizations ---------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//==-----------------------------------------------------------------------===//
#include "AMDILDevices.h"
#include "AMDGPUInstrInfo.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Constants.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionAnalysis.h"
#include "llvm/Function.h"
#include "llvm/Instructions.h"
#include "llvm/Module.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include <sstream>
#if 0
STATISTIC(PointerAssignments, "Number of dynamic pointer "
"assigments discovered");
STATISTIC(PointerSubtract, "Number of pointer subtractions discovered");
#endif
using namespace llvm;
// The Peephole optimization pass is used to do simple last minute optimizations
// that are required for correct code or to remove redundant functions
namespace {
class OpaqueType;
class LLVM_LIBRARY_VISIBILITY AMDGPUPeepholeOpt : public FunctionPass {
public:
TargetMachine &TM;
static char ID;
AMDGPUPeepholeOpt(TargetMachine &tm);
~AMDGPUPeepholeOpt();
const char *getPassName() const;
bool runOnFunction(Function &F);
bool doInitialization(Module &M);
bool doFinalization(Module &M);
void getAnalysisUsage(AnalysisUsage &AU) const;
protected:
private:
// Function to initiate all of the instruction level optimizations.
bool instLevelOptimizations(BasicBlock::iterator *inst);
// Quick check to see if we need to dump all of the pointers into the
// arena. If this is correct, then we set all pointers to exist in arena. This
// is a workaround for aliasing of pointers in a struct/union.
bool dumpAllIntoArena(Function &F);
// Because I don't want to invalidate any pointers while in the
// safeNestedForEachFunction. I push atomic conversions to a vector and handle
// it later. This function does the conversions if required.
void doAtomicConversionIfNeeded(Function &F);
// Because __amdil_is_constant cannot be properly evaluated if
// optimizations are disabled, the call's are placed in a vector
// and evaluated after the __amdil_image* functions are evaluated
// which should allow the __amdil_is_constant function to be
// evaluated correctly.
void doIsConstCallConversionIfNeeded();
bool mChanged;
bool mDebug;
bool mConvertAtomics;
CodeGenOpt::Level optLevel;
// Run a series of tests to see if we can optimize a CALL instruction.
bool optimizeCallInst(BasicBlock::iterator *bbb);
// A peephole optimization to optimize bit extract sequences.
bool optimizeBitExtract(Instruction *inst);
// A peephole optimization to optimize bit insert sequences.
bool optimizeBitInsert(Instruction *inst);
bool setupBitInsert(Instruction *base,
Instruction *&src,
Constant *&mask,
Constant *&shift);
// Expand the bit field insert instruction on versions of OpenCL that
// don't support it.
bool expandBFI(CallInst *CI);
// Expand the bit field mask instruction on version of OpenCL that
// don't support it.
bool expandBFM(CallInst *CI);
// On 7XX and 8XX operations, we do not have 24 bit signed operations. So in
// this case we need to expand them. These functions check for 24bit functions
// and then expand.
bool isSigned24BitOps(CallInst *CI);
void expandSigned24BitOps(CallInst *CI);
// One optimization that can occur is that if the required workgroup size is
// specified then the result of get_local_size is known at compile time and
// can be returned accordingly.
bool isRWGLocalOpt(CallInst *CI);
// On northern island cards, the division is slightly less accurate than on
// previous generations, so we need to utilize a more accurate division. So we
// can translate the accurate divide to a normal divide on all other cards.
bool convertAccurateDivide(CallInst *CI);
void expandAccurateDivide(CallInst *CI);
// If the alignment is set incorrectly, it can produce really inefficient
// code. This checks for this scenario and fixes it if possible.
bool correctMisalignedMemOp(Instruction *inst);
// If we are in no opt mode, then we need to make sure that
// local samplers are properly propagated as constant propagation
// doesn't occur and we need to know the value of kernel defined
// samplers at compile time.
bool propagateSamplerInst(CallInst *CI);
// Helper functions
// Group of functions that recursively calculate the size of a structure based
// on it's sub-types.
size_t getTypeSize(Type * const T, bool dereferencePtr = false);
size_t getTypeSize(StructType * const ST, bool dereferencePtr = false);
size_t getTypeSize(IntegerType * const IT, bool dereferencePtr = false);
size_t getTypeSize(FunctionType * const FT,bool dereferencePtr = false);
size_t getTypeSize(ArrayType * const AT, bool dereferencePtr = false);
size_t getTypeSize(VectorType * const VT, bool dereferencePtr = false);
size_t getTypeSize(PointerType * const PT, bool dereferencePtr = false);
size_t getTypeSize(OpaqueType * const OT, bool dereferencePtr = false);
LLVMContext *mCTX;
Function *mF;
const AMDGPUSubtarget *mSTM;
SmallVector< std::pair<CallInst *, Function *>, 16> atomicFuncs;
SmallVector<CallInst *, 16> isConstVec;
}; // class AMDGPUPeepholeOpt
char AMDGPUPeepholeOpt::ID = 0;
// A template function that has two levels of looping before calling the
// function with a pointer to the current iterator.
template<class InputIterator, class SecondIterator, class Function>
Function safeNestedForEach(InputIterator First, InputIterator Last,
SecondIterator S, Function F)
{
for ( ; First != Last; ++First) {
SecondIterator sf, sl;
for (sf = First->begin(), sl = First->end();
sf != sl; ) {
if (!F(&sf)) {
++sf;
}
}
}
return F;
}
} // anonymous namespace
namespace llvm {
FunctionPass *
createAMDGPUPeepholeOpt(TargetMachine &tm)
{
return new AMDGPUPeepholeOpt(tm);
}
} // llvm namespace
AMDGPUPeepholeOpt::AMDGPUPeepholeOpt(TargetMachine &tm)
: FunctionPass(ID), TM(tm)
{
mDebug = false;
optLevel = TM.getOptLevel();
}
AMDGPUPeepholeOpt::~AMDGPUPeepholeOpt()
{
}
const char *
AMDGPUPeepholeOpt::getPassName() const
{
return "AMDGPU PeepHole Optimization Pass";
}
bool
containsPointerType(Type *Ty)
{
if (!Ty) {
return false;
}
switch(Ty->getTypeID()) {
default:
return false;
case Type::StructTyID: {
const StructType *ST = dyn_cast<StructType>(Ty);
for (StructType::element_iterator stb = ST->element_begin(),
ste = ST->element_end(); stb != ste; ++stb) {
if (!containsPointerType(*stb)) {
continue;
}
return true;
}
break;
}
case Type::VectorTyID:
case Type::ArrayTyID:
return containsPointerType(dyn_cast<SequentialType>(Ty)->getElementType());
case Type::PointerTyID:
return true;
};
return false;
}
bool
AMDGPUPeepholeOpt::dumpAllIntoArena(Function &F)
{
bool dumpAll = false;
for (Function::const_arg_iterator cab = F.arg_begin(),
cae = F.arg_end(); cab != cae; ++cab) {
const Argument *arg = cab;
const PointerType *PT = dyn_cast<PointerType>(arg->getType());
if (!PT) {
continue;
}
Type *DereferencedType = PT->getElementType();
if (!dyn_cast<StructType>(DereferencedType)
) {
continue;
}
if (!containsPointerType(DereferencedType)) {
continue;
}
// FIXME: Because a pointer inside of a struct/union may be aliased to
// another pointer we need to take the conservative approach and place all
// pointers into the arena until more advanced detection is implemented.
dumpAll = true;
}
return dumpAll;
}
void
AMDGPUPeepholeOpt::doIsConstCallConversionIfNeeded()
{
if (isConstVec.empty()) {
return;
}
for (unsigned x = 0, y = isConstVec.size(); x < y; ++x) {
CallInst *CI = isConstVec[x];
Constant *CV = dyn_cast<Constant>(CI->getOperand(0));
Type *aType = Type::getInt32Ty(*mCTX);
Value *Val = (CV != NULL) ? ConstantInt::get(aType, 1)
: ConstantInt::get(aType, 0);
CI->replaceAllUsesWith(Val);
CI->eraseFromParent();
}
isConstVec.clear();
}
void
AMDGPUPeepholeOpt::doAtomicConversionIfNeeded(Function &F)
{
// Don't do anything if we don't have any atomic operations.
if (atomicFuncs.empty()) {
return;
}
// Change the function name for the atomic if it is required
uint32_t size = atomicFuncs.size();
for (uint32_t x = 0; x < size; ++x) {
atomicFuncs[x].first->setOperand(
atomicFuncs[x].first->getNumOperands()-1,
atomicFuncs[x].second);
}
mChanged = true;
if (mConvertAtomics) {
return;
}
}
bool
AMDGPUPeepholeOpt::runOnFunction(Function &MF)
{
mChanged = false;
mF = &MF;
mSTM = &TM.getSubtarget<AMDGPUSubtarget>();
if (mDebug) {
MF.dump();
}
mCTX = &MF.getType()->getContext();
mConvertAtomics = true;
safeNestedForEach(MF.begin(), MF.end(), MF.begin()->begin(),
std::bind1st(std::mem_fun(&AMDGPUPeepholeOpt::instLevelOptimizations),
this));
doAtomicConversionIfNeeded(MF);
doIsConstCallConversionIfNeeded();
if (mDebug) {
MF.dump();
}
return mChanged;
}
bool
AMDGPUPeepholeOpt::optimizeCallInst(BasicBlock::iterator *bbb)
{
Instruction *inst = (*bbb);
CallInst *CI = dyn_cast<CallInst>(inst);
if (!CI) {
return false;
}
if (isSigned24BitOps(CI)) {
expandSigned24BitOps(CI);
++(*bbb);
CI->eraseFromParent();
return true;
}
if (propagateSamplerInst(CI)) {
return false;
}
if (expandBFI(CI) || expandBFM(CI)) {
++(*bbb);
CI->eraseFromParent();
return true;
}
if (convertAccurateDivide(CI)) {
expandAccurateDivide(CI);
++(*bbb);
CI->eraseFromParent();
return true;
}
StringRef calleeName = CI->getOperand(CI->getNumOperands()-1)->getName();
if (calleeName.startswith("__amdil_is_constant")) {
// If we do not have optimizations, then this
// cannot be properly evaluated, so we add the
// call instruction to a vector and process
// them at the end of processing after the
// samplers have been correctly handled.
if (optLevel == CodeGenOpt::None) {
isConstVec.push_back(CI);
return false;
} else {
Constant *CV = dyn_cast<Constant>(CI->getOperand(0));
Type *aType = Type::getInt32Ty(*mCTX);
Value *Val = (CV != NULL) ? ConstantInt::get(aType, 1)
: ConstantInt::get(aType, 0);
CI->replaceAllUsesWith(Val);
++(*bbb);
CI->eraseFromParent();
return true;
}
}
if (calleeName.equals("__amdil_is_asic_id_i32")) {
ConstantInt *CV = dyn_cast<ConstantInt>(CI->getOperand(0));
Type *aType = Type::getInt32Ty(*mCTX);
Value *Val = CV;
if (Val) {
Val = ConstantInt::get(aType,
mSTM->device()->getDeviceFlag() & CV->getZExtValue());
} else {
Val = ConstantInt::get(aType, 0);
}
CI->replaceAllUsesWith(Val);
++(*bbb);
CI->eraseFromParent();
return true;
}
Function *F = dyn_cast<Function>(CI->getOperand(CI->getNumOperands()-1));
if (!F) {
return false;
}
if (F->getName().startswith("__atom") && !CI->getNumUses()
&& F->getName().find("_xchg") == StringRef::npos) {
std::string buffer(F->getName().str() + "_noret");
F = dyn_cast<Function>(
F->getParent()->getOrInsertFunction(buffer, F->getFunctionType()));
atomicFuncs.push_back(std::make_pair <CallInst*, Function*>(CI, F));
}
if (!mSTM->device()->isSupported(AMDGPUDeviceInfo::ArenaSegment)
&& !mSTM->device()->isSupported(AMDGPUDeviceInfo::MultiUAV)) {
return false;
}
if (!mConvertAtomics) {
return false;
}
StringRef name = F->getName();
if (name.startswith("__atom") && name.find("_g") != StringRef::npos) {
mConvertAtomics = false;
}
return false;
}
bool
AMDGPUPeepholeOpt::setupBitInsert(Instruction *base,
Instruction *&src,
Constant *&mask,
Constant *&shift)
{
if (!base) {
if (mDebug) {
dbgs() << "Null pointer passed into function.\n";
}
return false;
}
bool andOp = false;
if (base->getOpcode() == Instruction::Shl) {
shift = dyn_cast<Constant>(base->getOperand(1));
} else if (base->getOpcode() == Instruction::And) {
mask = dyn_cast<Constant>(base->getOperand(1));
andOp = true;
} else {
if (mDebug) {
dbgs() << "Failed setup with no Shl or And instruction on base opcode!\n";
}
// If the base is neither a Shl or a And, we don't fit any of the patterns above.
return false;
}
src = dyn_cast<Instruction>(base->getOperand(0));
if (!src) {
if (mDebug) {
dbgs() << "Failed setup since the base operand is not an instruction!\n";
}
return false;
}
// If we find an 'and' operation, then we don't need to
// find the next operation as we already know the
// bits that are valid at this point.
if (andOp) {
return true;
}
if (src->getOpcode() == Instruction::Shl && !shift) {
shift = dyn_cast<Constant>(src->getOperand(1));
src = dyn_cast<Instruction>(src->getOperand(0));
} else if (src->getOpcode() == Instruction::And && !mask) {
mask = dyn_cast<Constant>(src->getOperand(1));
}
if (!mask && !shift) {
if (mDebug) {
dbgs() << "Failed setup since both mask and shift are NULL!\n";
}
// Did not find a constant mask or a shift.
return false;
}
return true;
}
bool
AMDGPUPeepholeOpt::optimizeBitInsert(Instruction *inst)
{
if (!inst) {
return false;
}
if (!inst->isBinaryOp()) {
return false;
}
if (inst->getOpcode() != Instruction::Or) {
return false;
}
if (optLevel == CodeGenOpt::None) {
return false;
}
// We want to do an optimization on a sequence of ops that in the end equals a
// single ISA instruction.
// The base pattern for this optimization is - ((A & B) << C) | ((D & E) << F)
// Some simplified versions of this pattern are as follows:
// (A & B) | (D & E) when B & E == 0 && C == 0 && F == 0
// ((A & B) << C) | (D & E) when B ^ E == 0 && (1 << C) >= E
// (A & B) | ((D & E) << F) when B ^ E == 0 && (1 << F) >= B
// (A & B) | (D << F) when (1 << F) >= B
// (A << C) | (D & E) when (1 << C) >= E
if (mSTM->device()->getGeneration() == AMDGPUDeviceInfo::HD4XXX) {
// The HD4XXX hardware doesn't support the ubit_insert instruction.
return false;
}
Type *aType = inst->getType();
bool isVector = aType->isVectorTy();
int numEle = 1;
// This optimization only works on 32bit integers.
if (aType->getScalarType()
!= Type::getInt32Ty(inst->getContext())) {
return false;
}
if (isVector) {
const VectorType *VT = dyn_cast<VectorType>(aType);
numEle = VT->getNumElements();
// We currently cannot support more than 4 elements in a intrinsic and we
// cannot support Vec3 types.
if (numEle > 4 || numEle == 3) {
return false;
}
}
// TODO: Handle vectors.
if (isVector) {
if (mDebug) {
dbgs() << "!!! Vectors are not supported yet!\n";
}
return false;
}
Instruction *LHSSrc = NULL, *RHSSrc = NULL;
Constant *LHSMask = NULL, *RHSMask = NULL;
Constant *LHSShift = NULL, *RHSShift = NULL;
Instruction *LHS = dyn_cast<Instruction>(inst->getOperand(0));
Instruction *RHS = dyn_cast<Instruction>(inst->getOperand(1));
if (!setupBitInsert(LHS, LHSSrc, LHSMask, LHSShift)) {
if (mDebug) {
dbgs() << "Found an OR Operation that failed setup!\n";
inst->dump();
if (LHS) { LHS->dump(); }
if (LHSSrc) { LHSSrc->dump(); }
if (LHSMask) { LHSMask->dump(); }
if (LHSShift) { LHSShift->dump(); }
}
// There was an issue with the setup for BitInsert.
return false;
}
if (!setupBitInsert(RHS, RHSSrc, RHSMask, RHSShift)) {
if (mDebug) {
dbgs() << "Found an OR Operation that failed setup!\n";
inst->dump();
if (RHS) { RHS->dump(); }
if (RHSSrc) { RHSSrc->dump(); }
if (RHSMask) { RHSMask->dump(); }
if (RHSShift) { RHSShift->dump(); }
}
// There was an issue with the setup for BitInsert.
return false;
}
if (mDebug) {
dbgs() << "Found an OR operation that can possible be optimized to ubit insert!\n";
dbgs() << "Op: "; inst->dump();
dbgs() << "LHS: "; if (LHS) { LHS->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "LHS Src: "; if (LHSSrc) { LHSSrc->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "LHS Mask: "; if (LHSMask) { LHSMask->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "LHS Shift: "; if (LHSShift) { LHSShift->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "RHS: "; if (RHS) { RHS->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "RHS Src: "; if (RHSSrc) { RHSSrc->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "RHS Mask: "; if (RHSMask) { RHSMask->dump(); } else { dbgs() << "(None)\n"; }
dbgs() << "RHS Shift: "; if (RHSShift) { RHSShift->dump(); } else { dbgs() << "(None)\n"; }
}
Constant *offset = NULL;
Constant *width = NULL;
int32_t lhsMaskVal = 0, rhsMaskVal = 0;
int32_t lhsShiftVal = 0, rhsShiftVal = 0;
int32_t lhsMaskWidth = 0, rhsMaskWidth = 0;
int32_t lhsMaskOffset = 0, rhsMaskOffset = 0;
lhsMaskVal = (int32_t)(LHSMask
? dyn_cast<ConstantInt>(LHSMask)->getZExtValue() : 0);
rhsMaskVal = (int32_t)(RHSMask
? dyn_cast<ConstantInt>(RHSMask)->getZExtValue() : 0);
lhsShiftVal = (int32_t)(LHSShift
? dyn_cast<ConstantInt>(LHSShift)->getZExtValue() : 0);
rhsShiftVal = (int32_t)(RHSShift
? dyn_cast<ConstantInt>(RHSShift)->getZExtValue() : 0);
lhsMaskWidth = lhsMaskVal ? CountPopulation_32(lhsMaskVal) : 32 - lhsShiftVal;
rhsMaskWidth = rhsMaskVal ? CountPopulation_32(rhsMaskVal) : 32 - rhsShiftVal;
lhsMaskOffset = lhsMaskVal ? CountTrailingZeros_32(lhsMaskVal) : lhsShiftVal;
rhsMaskOffset = rhsMaskVal ? CountTrailingZeros_32(rhsMaskVal) : rhsShiftVal;
// TODO: Handle the case of A & B | D & ~B(i.e. inverted masks).
if (mDebug) {
dbgs() << "Found pattern: \'((A" << (LHSMask ? " & B)" : ")");
dbgs() << (LHSShift ? " << C)" : ")") << " | ((D" ;
dbgs() << (RHSMask ? " & E)" : ")");
dbgs() << (RHSShift ? " << F)\'\n" : ")\'\n");
dbgs() << "A = LHSSrc\t\tD = RHSSrc \n";
dbgs() << "B = " << lhsMaskVal << "\t\tE = " << rhsMaskVal << "\n";
dbgs() << "C = " << lhsShiftVal << "\t\tF = " << rhsShiftVal << "\n";
dbgs() << "width(B) = " << lhsMaskWidth;
dbgs() << "\twidth(E) = " << rhsMaskWidth << "\n";
dbgs() << "offset(B) = " << lhsMaskOffset;
dbgs() << "\toffset(E) = " << rhsMaskOffset << "\n";
dbgs() << "Constraints: \n";
dbgs() << "\t(1) B ^ E == 0\n";
dbgs() << "\t(2-LHS) B is a mask\n";
dbgs() << "\t(2-LHS) E is a mask\n";
dbgs() << "\t(3-LHS) (offset(B)) >= (width(E) + offset(E))\n";
dbgs() << "\t(3-RHS) (offset(E)) >= (width(B) + offset(B))\n";
}
if ((lhsMaskVal || rhsMaskVal) && !(lhsMaskVal ^ rhsMaskVal)) {
if (mDebug) {
dbgs() << lhsMaskVal << " ^ " << rhsMaskVal;
dbgs() << " = " << (lhsMaskVal ^ rhsMaskVal) << "\n";
dbgs() << "Failed constraint 1!\n";
}
return false;
}
if (mDebug) {
dbgs() << "LHS = " << lhsMaskOffset << "";
dbgs() << " >= (" << rhsMaskWidth << " + " << rhsMaskOffset << ") = ";
dbgs() << (lhsMaskOffset >= (rhsMaskWidth + rhsMaskOffset));
dbgs() << "\nRHS = " << rhsMaskOffset << "";
dbgs() << " >= (" << lhsMaskWidth << " + " << lhsMaskOffset << ") = ";
dbgs() << (rhsMaskOffset >= (lhsMaskWidth + lhsMaskOffset));
dbgs() << "\n";
}
if (lhsMaskOffset >= (rhsMaskWidth + rhsMaskOffset)) {
offset = ConstantInt::get(aType, lhsMaskOffset, false);
width = ConstantInt::get(aType, lhsMaskWidth, false);
RHSSrc = RHS;
if (!isMask_32(lhsMaskVal) && !isShiftedMask_32(lhsMaskVal)) {
if (mDebug) {
dbgs() << "Value is not a Mask: " << lhsMaskVal << "\n";
dbgs() << "Failed constraint 2!\n";
}
return false;
}
if (!LHSShift) {
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
"MaskShr", LHS);
} else if (lhsShiftVal != lhsMaskOffset) {
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
"MaskShr", LHS);
}
if (mDebug) {
dbgs() << "Optimizing LHS!\n";
}
} else if (rhsMaskOffset >= (lhsMaskWidth + lhsMaskOffset)) {
offset = ConstantInt::get(aType, rhsMaskOffset, false);
width = ConstantInt::get(aType, rhsMaskWidth, false);
LHSSrc = RHSSrc;
RHSSrc = LHS;
if (!isMask_32(rhsMaskVal) && !isShiftedMask_32(rhsMaskVal)) {
if (mDebug) {
dbgs() << "Non-Mask: " << rhsMaskVal << "\n";
dbgs() << "Failed constraint 2!\n";
}
return false;
}
if (!RHSShift) {
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
"MaskShr", RHS);
} else if (rhsShiftVal != rhsMaskOffset) {
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
"MaskShr", RHS);
}
if (mDebug) {
dbgs() << "Optimizing RHS!\n";
}
} else {
if (mDebug) {
dbgs() << "Failed constraint 3!\n";
}
return false;
}
if (mDebug) {
dbgs() << "Width: "; if (width) { width->dump(); } else { dbgs() << "(0)\n"; }
dbgs() << "Offset: "; if (offset) { offset->dump(); } else { dbgs() << "(0)\n"; }
dbgs() << "LHSSrc: "; if (LHSSrc) { LHSSrc->dump(); } else { dbgs() << "(0)\n"; }
dbgs() << "RHSSrc: "; if (RHSSrc) { RHSSrc->dump(); } else { dbgs() << "(0)\n"; }
}
if (!offset || !width) {
if (mDebug) {
dbgs() << "Either width or offset are NULL, failed detection!\n";
}
return false;
}
// Lets create the function signature.
std::vector<Type *> callTypes;
callTypes.push_back(aType);
callTypes.push_back(aType);
callTypes.push_back(aType);
callTypes.push_back(aType);
FunctionType *funcType = FunctionType::get(aType, callTypes, false);
std::string name = "__amdil_ubit_insert";
if (isVector) { name += "_v" + itostr(numEle) + "u32"; } else { name += "_u32"; }
Function *Func =
dyn_cast<Function>(inst->getParent()->getParent()->getParent()->
getOrInsertFunction(llvm::StringRef(name), funcType));
Value *Operands[4] = {
width,
offset,
LHSSrc,
RHSSrc
};
CallInst *CI = CallInst::Create(Func, Operands, "BitInsertOpt");
if (mDebug) {
dbgs() << "Old Inst: ";
inst->dump();
dbgs() << "New Inst: ";
CI->dump();
dbgs() << "\n\n";
}
CI->insertBefore(inst);
inst->replaceAllUsesWith(CI);
return true;
}
bool
AMDGPUPeepholeOpt::optimizeBitExtract(Instruction *inst)
{
if (!inst) {
return false;
}
if (!inst->isBinaryOp()) {
return false;
}
if (inst->getOpcode() != Instruction::And) {
return false;
}
if (optLevel == CodeGenOpt::None) {
return false;
}
// We want to do some simple optimizations on Shift right/And patterns. The
// basic optimization is to turn (A >> B) & C where A is a 32bit type, B is a
// value smaller than 32 and C is a mask. If C is a constant value, then the
// following transformation can occur. For signed integers, it turns into the
// function call dst = __amdil_ibit_extract(log2(C), B, A) For unsigned
// integers, it turns into the function call dst =
// __amdil_ubit_extract(log2(C), B, A) The function __amdil_[u|i]bit_extract
// can be found in Section 7.9 of the ATI IL spec of the stream SDK for
// Evergreen hardware.
if (mSTM->device()->getGeneration() == AMDGPUDeviceInfo::HD4XXX) {
// This does not work on HD4XXX hardware.
return false;
}
Type *aType = inst->getType();
bool isVector = aType->isVectorTy();
// XXX Support vector types
if (isVector) {
return false;
}
int numEle = 1;
// This only works on 32bit integers
if (aType->getScalarType()
!= Type::getInt32Ty(inst->getContext())) {
return false;
}
if (isVector) {
const VectorType *VT = dyn_cast<VectorType>(aType);
numEle = VT->getNumElements();
// We currently cannot support more than 4 elements in a intrinsic and we
// cannot support Vec3 types.
if (numEle > 4 || numEle == 3) {
return false;
}
}
BinaryOperator *ShiftInst = dyn_cast<BinaryOperator>(inst->getOperand(0));
// If the first operand is not a shift instruction, then we can return as it
// doesn't match this pattern.
if (!ShiftInst || !ShiftInst->isShift()) {
return false;
}
// If we are a shift left, then we need don't match this pattern.
if (ShiftInst->getOpcode() == Instruction::Shl) {
return false;
}
bool isSigned = ShiftInst->isArithmeticShift();
Constant *AndMask = dyn_cast<Constant>(inst->getOperand(1));
Constant *ShrVal = dyn_cast<Constant>(ShiftInst->getOperand(1));
// Lets make sure that the shift value and the and mask are constant integers.
if (!AndMask || !ShrVal) {
return false;
}
Constant *newMaskConst;
Constant *shiftValConst;
if (isVector) {
// Handle the vector case
std::vector<Constant *> maskVals;
std::vector<Constant *> shiftVals;
ConstantVector *AndMaskVec = dyn_cast<ConstantVector>(AndMask);
ConstantVector *ShrValVec = dyn_cast<ConstantVector>(ShrVal);
Type *scalarType = AndMaskVec->getType()->getScalarType();
assert(AndMaskVec->getNumOperands() ==
ShrValVec->getNumOperands() && "cannot have a "
"combination where the number of elements to a "
"shift and an and are different!");
for (size_t x = 0, y = AndMaskVec->getNumOperands(); x < y; ++x) {
ConstantInt *AndCI = dyn_cast<ConstantInt>(AndMaskVec->getOperand(x));
ConstantInt *ShiftIC = dyn_cast<ConstantInt>(ShrValVec->getOperand(x));
if (!AndCI || !ShiftIC) {
return false;
}
uint32_t maskVal = (uint32_t)AndCI->getZExtValue();
if (!isMask_32(maskVal)) {
return false;
}
maskVal = (uint32_t)CountTrailingOnes_32(maskVal);
uint32_t shiftVal = (uint32_t)ShiftIC->getZExtValue();
// If the mask or shiftval is greater than the bitcount, then break out.
if (maskVal >= 32 || shiftVal >= 32) {
return false;
}
// If the mask val is greater than the the number of original bits left
// then this optimization is invalid.
if (maskVal > (32 - shiftVal)) {
return false;
}
maskVals.push_back(ConstantInt::get(scalarType, maskVal, isSigned));
shiftVals.push_back(ConstantInt::get(scalarType, shiftVal, isSigned));
}
newMaskConst = ConstantVector::get(maskVals);
shiftValConst = ConstantVector::get(shiftVals);
} else {
// Handle the scalar case
uint32_t maskVal = (uint32_t)dyn_cast<ConstantInt>(AndMask)->getZExtValue();
// This must be a mask value where all lower bits are set to 1 and then any
// bit higher is set to 0.
if (!isMask_32(maskVal)) {
return false;
}
maskVal = (uint32_t)CountTrailingOnes_32(maskVal);
// Count the number of bits set in the mask, this is the width of the
// resulting bit set that is extracted from the source value.
uint32_t shiftVal = (uint32_t)dyn_cast<ConstantInt>(ShrVal)->getZExtValue();
// If the mask or shift val is greater than the bitcount, then break out.
if (maskVal >= 32 || shiftVal >= 32) {
return false;
}
// If the mask val is greater than the the number of original bits left then
// this optimization is invalid.
if (maskVal > (32 - shiftVal)) {
return false;
}
newMaskConst = ConstantInt::get(aType, maskVal, isSigned);
shiftValConst = ConstantInt::get(aType, shiftVal, isSigned);
}
// Lets create the function signature.
std::vector<Type *> callTypes;
callTypes.push_back(aType);
callTypes.push_back(aType);
callTypes.push_back(aType);
FunctionType *funcType = FunctionType::get(aType, callTypes, false);
std::string name = "llvm.AMDIL.bit.extract.u32";
if (isVector) {
name += ".v" + itostr(numEle) + "i32";
} else {
name += ".";
}
// Lets create the function.
Function *Func =
dyn_cast<Function>(inst->getParent()->getParent()->getParent()->
getOrInsertFunction(llvm::StringRef(name), funcType));
Value *Operands[3] = {
ShiftInst->getOperand(0),
shiftValConst,
newMaskConst
};
// Lets create the Call with the operands
CallInst *CI = CallInst::Create(Func, Operands, "ByteExtractOpt");
CI->setDoesNotAccessMemory();
CI->insertBefore(inst);
inst->replaceAllUsesWith(CI);
return true;
}
bool
AMDGPUPeepholeOpt::expandBFI(CallInst *CI)
{
if (!CI) {
return false;
}
Value *LHS = CI->getOperand(CI->getNumOperands() - 1);
if (!LHS->getName().startswith("__amdil_bfi")) {
return false;
}
Type* type = CI->getOperand(0)->getType();
Constant *negOneConst = NULL;
if (type->isVectorTy()) {
std::vector<Constant *> negOneVals;
negOneConst = ConstantInt::get(CI->getContext(),
APInt(32, StringRef("-1"), 10));
for (size_t x = 0,
y = dyn_cast<VectorType>(type)->getNumElements(); x < y; ++x) {
negOneVals.push_back(negOneConst);
}
negOneConst = ConstantVector::get(negOneVals);
} else {
negOneConst = ConstantInt::get(CI->getContext(),
APInt(32, StringRef("-1"), 10));
}
// __amdil_bfi => (A & B) | (~A & C)
BinaryOperator *lhs =
BinaryOperator::Create(Instruction::And, CI->getOperand(0),
CI->getOperand(1), "bfi_and", CI);
BinaryOperator *rhs =
BinaryOperator::Create(Instruction::Xor, CI->getOperand(0), negOneConst,
"bfi_not", CI);
rhs = BinaryOperator::Create(Instruction::And, rhs, CI->getOperand(2),
"bfi_and", CI);
lhs = BinaryOperator::Create(Instruction::Or, lhs, rhs, "bfi_or", CI);
CI->replaceAllUsesWith(lhs);
return true;
}
bool
AMDGPUPeepholeOpt::expandBFM(CallInst *CI)
{
if (!CI) {
return false;
}
Value *LHS = CI->getOperand(CI->getNumOperands() - 1);
if (!LHS->getName().startswith("__amdil_bfm")) {
return false;
}
// __amdil_bfm => ((1 << (src0 & 0x1F)) - 1) << (src1 & 0x1f)
Constant *newMaskConst = NULL;
Constant *newShiftConst = NULL;
Type* type = CI->getOperand(0)->getType();
if (type->isVectorTy()) {
std::vector<Constant*> newMaskVals, newShiftVals;
newMaskConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 0x1F);
newShiftConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 1);
for (size_t x = 0,
y = dyn_cast<VectorType>(type)->getNumElements(); x < y; ++x) {
newMaskVals.push_back(newMaskConst);
newShiftVals.push_back(newShiftConst);
}
newMaskConst = ConstantVector::get(newMaskVals);
newShiftConst = ConstantVector::get(newShiftVals);
} else {
newMaskConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 0x1F);
newShiftConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 1);
}
BinaryOperator *lhs =
BinaryOperator::Create(Instruction::And, CI->getOperand(0),
newMaskConst, "bfm_mask", CI);
lhs = BinaryOperator::Create(Instruction::Shl, newShiftConst,
lhs, "bfm_shl", CI);
lhs = BinaryOperator::Create(Instruction::Sub, lhs,
newShiftConst, "bfm_sub", CI);
BinaryOperator *rhs =
BinaryOperator::Create(Instruction::And, CI->getOperand(1),
newMaskConst, "bfm_mask", CI);
lhs = BinaryOperator::Create(Instruction::Shl, lhs, rhs, "bfm_shl", CI);
CI->replaceAllUsesWith(lhs);
return true;
}
bool
AMDGPUPeepholeOpt::instLevelOptimizations(BasicBlock::iterator *bbb)
{
Instruction *inst = (*bbb);
if (optimizeCallInst(bbb)) {
return true;
}
if (optimizeBitExtract(inst)) {
return false;
}
if (optimizeBitInsert(inst)) {
return false;
}
if (correctMisalignedMemOp(inst)) {
return false;
}
return false;
}
bool
AMDGPUPeepholeOpt::correctMisalignedMemOp(Instruction *inst)
{
LoadInst *linst = dyn_cast<LoadInst>(inst);
StoreInst *sinst = dyn_cast<StoreInst>(inst);
unsigned alignment;
Type* Ty = inst->getType();
if (linst) {
alignment = linst->getAlignment();
Ty = inst->getType();
} else if (sinst) {
alignment = sinst->getAlignment();
Ty = sinst->getValueOperand()->getType();
} else {
return false;
}
unsigned size = getTypeSize(Ty);
if (size == alignment || size < alignment) {
return false;
}
if (!Ty->isStructTy()) {
return false;
}
if (alignment < 4) {
if (linst) {
linst->setAlignment(0);
return true;
} else if (sinst) {
sinst->setAlignment(0);
return true;
}
}
return false;
}
bool
AMDGPUPeepholeOpt::isSigned24BitOps(CallInst *CI)
{
if (!CI) {
return false;
}
Value *LHS = CI->getOperand(CI->getNumOperands() - 1);
std::string namePrefix = LHS->getName().substr(0, 14);
if (namePrefix != "__amdil_imad24" && namePrefix != "__amdil_imul24"
&& namePrefix != "__amdil__imul24_high") {
return false;
}
if (mSTM->device()->usesHardware(AMDGPUDeviceInfo::Signed24BitOps)) {
return false;
}
return true;
}
void
AMDGPUPeepholeOpt::expandSigned24BitOps(CallInst *CI)
{
assert(isSigned24BitOps(CI) && "Must be a "
"signed 24 bit operation to call this function!");
Value *LHS = CI->getOperand(CI->getNumOperands()-1);
// On 7XX and 8XX we do not have signed 24bit, so we need to
// expand it to the following:
// imul24 turns into 32bit imul
// imad24 turns into 32bit imad
// imul24_high turns into 32bit imulhigh
if (LHS->getName().substr(0, 14) == "__amdil_imad24") {
Type *aType = CI->getOperand(0)->getType();
bool isVector = aType->isVectorTy();
int numEle = isVector ? dyn_cast<VectorType>(aType)->getNumElements() : 1;
std::vector<Type*> callTypes;
callTypes.push_back(CI->getOperand(0)->getType());
callTypes.push_back(CI->getOperand(1)->getType());
callTypes.push_back(CI->getOperand(2)->getType());
FunctionType *funcType =
FunctionType::get(CI->getOperand(0)->getType(), callTypes, false);
std::string name = "__amdil_imad";
if (isVector) {
name += "_v" + itostr(numEle) + "i32";
} else {
name += "_i32";
}
Function *Func = dyn_cast<Function>(
CI->getParent()->getParent()->getParent()->
getOrInsertFunction(llvm::StringRef(name), funcType));
Value *Operands[3] = {
CI->getOperand(0),
CI->getOperand(1),
CI->getOperand(2)
};
CallInst *nCI = CallInst::Create(Func, Operands, "imad24");
nCI->insertBefore(CI);
CI->replaceAllUsesWith(nCI);
} else if (LHS->getName().substr(0, 14) == "__amdil_imul24") {
BinaryOperator *mulOp =
BinaryOperator::Create(Instruction::Mul, CI->getOperand(0),
CI->getOperand(1), "imul24", CI);
CI->replaceAllUsesWith(mulOp);
} else if (LHS->getName().substr(0, 19) == "__amdil_imul24_high") {
Type *aType = CI->getOperand(0)->getType();
bool isVector = aType->isVectorTy();
int numEle = isVector ? dyn_cast<VectorType>(aType)->getNumElements() : 1;
std::vector<Type*> callTypes;
callTypes.push_back(CI->getOperand(0)->getType());
callTypes.push_back(CI->getOperand(1)->getType());
FunctionType *funcType =
FunctionType::get(CI->getOperand(0)->getType(), callTypes, false);
std::string name = "__amdil_imul_high";
if (isVector) {
name += "_v" + itostr(numEle) + "i32";
} else {
name += "_i32";
}
Function *Func = dyn_cast<Function>(
CI->getParent()->getParent()->getParent()->
getOrInsertFunction(llvm::StringRef(name), funcType));
Value *Operands[2] = {
CI->getOperand(0),
CI->getOperand(1)
};
CallInst *nCI = CallInst::Create(Func, Operands, "imul24_high");
nCI->insertBefore(CI);
CI->replaceAllUsesWith(nCI);
}
}
bool
AMDGPUPeepholeOpt::isRWGLocalOpt(CallInst *CI)
{
return (CI != NULL
&& CI->getOperand(CI->getNumOperands() - 1)->getName()
== "__amdil_get_local_size_int");
}
bool
AMDGPUPeepholeOpt::convertAccurateDivide(CallInst *CI)
{
if (!CI) {
return false;
}
if (mSTM->device()->getGeneration() == AMDGPUDeviceInfo::HD6XXX
&& (mSTM->getDeviceName() == "cayman")) {
return false;
}
return CI->getOperand(CI->getNumOperands() - 1)->getName().substr(0, 20)
== "__amdil_improved_div";
}
void
AMDGPUPeepholeOpt::expandAccurateDivide(CallInst *CI)
{
assert(convertAccurateDivide(CI)
&& "expanding accurate divide can only happen if it is expandable!");
BinaryOperator *divOp =
BinaryOperator::Create(Instruction::FDiv, CI->getOperand(0),
CI->getOperand(1), "fdiv32", CI);
CI->replaceAllUsesWith(divOp);
}
bool
AMDGPUPeepholeOpt::propagateSamplerInst(CallInst *CI)
{
if (optLevel != CodeGenOpt::None) {
return false;
}
if (!CI) {
return false;
}
unsigned funcNameIdx = 0;
funcNameIdx = CI->getNumOperands() - 1;
StringRef calleeName = CI->getOperand(funcNameIdx)->getName();
if (calleeName != "__amdil_image2d_read_norm"
&& calleeName != "__amdil_image2d_read_unnorm"
&& calleeName != "__amdil_image3d_read_norm"
&& calleeName != "__amdil_image3d_read_unnorm") {
return false;
}
unsigned samplerIdx = 2;
samplerIdx = 1;
Value *sampler = CI->getOperand(samplerIdx);
LoadInst *lInst = dyn_cast<LoadInst>(sampler);
if (!lInst) {
return false;
}
if (lInst->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
return false;
}
GlobalVariable *gv = dyn_cast<GlobalVariable>(lInst->getPointerOperand());
// If we are loading from what is not a global value, then we
// fail and return.
if (!gv) {
return false;
}
// If we don't have an initializer or we have an initializer and
// the initializer is not a 32bit integer, we fail.
if (!gv->hasInitializer()
|| !gv->getInitializer()->getType()->isIntegerTy(32)) {
return false;
}
// Now that we have the global variable initializer, lets replace
// all uses of the load instruction with the samplerVal and
// reparse the __amdil_is_constant() function.
Constant *samplerVal = gv->getInitializer();
lInst->replaceAllUsesWith(samplerVal);
return true;
}
bool
AMDGPUPeepholeOpt::doInitialization(Module &M)
{
return false;
}
bool
AMDGPUPeepholeOpt::doFinalization(Module &M)
{
return false;
}
void
AMDGPUPeepholeOpt::getAnalysisUsage(AnalysisUsage &AU) const
{
AU.addRequired<MachineFunctionAnalysis>();
FunctionPass::getAnalysisUsage(AU);
AU.setPreservesAll();
}
size_t AMDGPUPeepholeOpt::getTypeSize(Type * const T, bool dereferencePtr) {
size_t size = 0;
if (!T) {
return size;
}
switch (T->getTypeID()) {
case Type::X86_FP80TyID:
case Type::FP128TyID:
case Type::PPC_FP128TyID:
case Type::LabelTyID:
assert(0 && "These types are not supported by this backend");
default:
case Type::FloatTyID:
case Type::DoubleTyID:
size = T->getPrimitiveSizeInBits() >> 3;
break;
case Type::PointerTyID:
size = getTypeSize(dyn_cast<PointerType>(T), dereferencePtr);
break;
case Type::IntegerTyID:
size = getTypeSize(dyn_cast<IntegerType>(T), dereferencePtr);
break;
case Type::StructTyID:
size = getTypeSize(dyn_cast<StructType>(T), dereferencePtr);
break;
case Type::ArrayTyID:
size = getTypeSize(dyn_cast<ArrayType>(T), dereferencePtr);
break;
case Type::FunctionTyID:
size = getTypeSize(dyn_cast<FunctionType>(T), dereferencePtr);
break;
case Type::VectorTyID:
size = getTypeSize(dyn_cast<VectorType>(T), dereferencePtr);
break;
};
return size;
}
size_t AMDGPUPeepholeOpt::getTypeSize(StructType * const ST,
bool dereferencePtr) {
size_t size = 0;
if (!ST) {
return size;
}
Type *curType;
StructType::element_iterator eib;
StructType::element_iterator eie;
for (eib = ST->element_begin(), eie = ST->element_end(); eib != eie; ++eib) {
curType = *eib;
size += getTypeSize(curType, dereferencePtr);
}
return size;
}
size_t AMDGPUPeepholeOpt::getTypeSize(IntegerType * const IT,
bool dereferencePtr) {
return IT ? (IT->getBitWidth() >> 3) : 0;
}
size_t AMDGPUPeepholeOpt::getTypeSize(FunctionType * const FT,
bool dereferencePtr) {
assert(0 && "Should not be able to calculate the size of an function type");
return 0;
}
size_t AMDGPUPeepholeOpt::getTypeSize(ArrayType * const AT,
bool dereferencePtr) {
return (size_t)(AT ? (getTypeSize(AT->getElementType(),
dereferencePtr) * AT->getNumElements())
: 0);
}
size_t AMDGPUPeepholeOpt::getTypeSize(VectorType * const VT,
bool dereferencePtr) {
return VT ? (VT->getBitWidth() >> 3) : 0;
}
size_t AMDGPUPeepholeOpt::getTypeSize(PointerType * const PT,
bool dereferencePtr) {
if (!PT) {
return 0;
}
Type *CT = PT->getElementType();
if (CT->getTypeID() == Type::StructTyID &&
PT->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) {
return getTypeSize(dyn_cast<StructType>(CT));
} else if (dereferencePtr) {
size_t size = 0;
for (size_t x = 0, y = PT->getNumContainedTypes(); x < y; ++x) {
size += getTypeSize(PT->getContainedType(x), dereferencePtr);
}
return size;
} else {
return 4;
}
}
size_t AMDGPUPeepholeOpt::getTypeSize(OpaqueType * const OT,
bool dereferencePtr) {
//assert(0 && "Should not be able to calculate the size of an opaque type");
return 4;
}