VSadov/Satori
1
0
Fork 0
mirror of https://github.com/VSadov/Satori.git synced 2025-06-09 17:44:48 +09:00
Code Activity

Next round of multireg preliminary changes (#36155)

Browse source 
This is a zero-diff set of mostly refactoring changes in preparation for supporting multireg locals:
- Move `genRegCopy()` and `genStructReturn()` to codegencommon.cpp, making a new method for `genSIMDSplitReturn` which is target-specific.
- Factor out a new `genUnspillLocal` method from `genUnspillRegIfNeeded()`.
- Similarly factor out `genSpillLocal()`
- Rename `genMultiRegCallStoreToLocal()` and more generally support multireg local stores.
- Fix a bug in the order and shift amount for last-use bits on `GenTreeLclVar`
- Some additional cleanup and preparatory changes
This commit is contained in:
Carol Eidt 2020-05-15 13:04:26 -07:00 • committed by GitHub
parent b764cae5f8
commit ca0fd167d5
Signed by: github
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 731 additions and 976 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/coreclr/src/jit/codegen.h
View file

@ -1113,6 +1113,9 @@ protected:
// Do liveness update for register produced by the current node in codegen after
// code has been emitted for it.
void genProduceReg(GenTree* tree);
void genSpillLocal(unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum);
void genUnspillLocal(
unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse);
void genUnspillRegIfNeeded(GenTree* tree);
regNumber genConsumeReg(GenTree* tree);
void genCopyRegIfNeeded(GenTree* tree, regNumber needReg);
@ -1275,10 +1278,13 @@ protected:
void genEHFinallyOrFilterRet(BasicBlock* block);
#endif // !FEATURE_EH_FUNCLETS
void genMultiRegCallStoreToLocal(GenTree* treeNode);
void genMultiRegStoreToLocal(GenTree* treeNode);
// Deals with codegen for muti-register struct returns.
// Codegen for multi-register struct returns.
bool isStructReturn(GenTree* treeNode);
#ifdef FEATURE_SIMD
void genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc);
#endif
void genStructReturn(GenTree* treeNode);
#if defined(TARGET_X86) || defined(TARGET_ARM)

4
src/coreclr/src/jit/codegenarm.cpp
View file

@ -961,7 +961,7 @@ void CodeGen::genCodeForLclVar(GenTreeLclVar* tree)
// If this is a register candidate that has been spilled, genConsumeReg() will
// reload it at the point of use. Otherwise, if it's not in a register, we load it here.
if (!isRegCandidate && !(tree->gtFlags & GTF_SPILLED))
if (!isRegCandidate && !tree->IsMultiReg() && !(tree->gtFlags & GTF_SPILLED))
{
const LclVarDsc* varDsc = compiler->lvaGetDesc(tree);
var_types type = varDsc->GetRegisterType(tree);
@ -1050,7 +1050,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree)
// case is handled separately.
if (data->gtSkipReloadOrCopy()->IsMultiRegCall())
{
genMultiRegCallStoreToLocal(tree);
genMultiRegStoreToLocal(tree);
}
else
{

4
src/coreclr/src/jit/codegenarm64.cpp
View file

@ -1839,7 +1839,7 @@ void CodeGen::genCodeForLclVar(GenTreeLclVar* tree)
// If this is a register candidate that has been spilled, genConsumeReg() will
// reload it at the point of use. Otherwise, if it's not in a register, we load it here.
if (!isRegCandidate && !(tree->gtFlags & GTF_SPILLED))
if (!isRegCandidate && !tree->IsMultiReg() && !(tree->gtFlags & GTF_SPILLED))
{
// targetType must be a normal scalar type and not a TYP_STRUCT
assert(targetType != TYP_STRUCT);
@ -1929,7 +1929,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree)
// case is handled separately.
if (data->gtSkipReloadOrCopy()->IsMultiRegCall())
{
genMultiRegCallStoreToLocal(tree);
genMultiRegStoreToLocal(tree);
}
else
{

419
src/coreclr/src/jit/codegenarmarch.cpp
View file

@ -1355,7 +1355,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode)
#endif // FEATURE_ARG_SPLIT
//----------------------------------------------------------------------------------
// genMultiRegCallStoreToLocal: store multi-reg return value of a call node to a local
// genMultiRegStoreToLocal: store multi-reg return value of a call node to a local
//
// Arguments:
// treeNode - Gentree of GT_STORE_LCL_VAR
@ -1364,42 +1364,35 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode)
// None
//
// Assumption:
// The child of store is a multi-reg call node.
// genProduceReg() on treeNode is made by caller of this routine.
// The child of store is a multi-reg node.
//
void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
void CodeGen::genMultiRegStoreToLocal(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_STORE_LCL_VAR);
#if defined(TARGET_ARM)
// Longs are returned in two return registers on Arm32.
// Structs are returned in four registers on ARM32 and HFAs.
assert(varTypeIsLong(treeNode) || varTypeIsStruct(treeNode));
#elif defined(TARGET_ARM64)
// Structs of size >=9 and <=16 are returned in two return registers on ARM64 and HFAs.
assert(varTypeIsStruct(treeNode));
#endif // TARGET*
assert(varTypeIsStruct(treeNode) || varTypeIsMultiReg(treeNode));
GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
assert(op1->IsMultiRegNode());
unsigned regCount = op1->GetMultiRegCount();
// Assumption: current implementation requires that a multi-reg
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = &(compiler->lvaTable[lclNum]);
LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum);
if (op1->OperIs(GT_CALL))
{
assert(regCount <= MAX_RET_REG_COUNT);
noway_assert(varDsc->lvIsMultiRegRet);
GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
assert(call->HasMultiRegRetVal());
}
genConsumeRegs(op1);
const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc();
const unsigned regCount = pRetTypeDesc->GetReturnRegCount();
int offset = 0;
if (treeNode->GetRegNum() != REG_NA)
// Check for the case of an enregistered SIMD type that's returned in multiple registers.
if (varDsc->lvIsRegCandidate() && treeNode->GetRegNum() != REG_NA)
{
// Right now the only enregistrable multi-reg return types supported are SIMD types.
assert(varTypeIsSIMD(treeNode));
assert(regCount != 0);
@ -1409,8 +1402,8 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
// Insert pieces in reverse order
for (int i = regCount - 1; i >= 0; --i)
{
var_types type = pRetTypeDesc->GetReturnRegType(i);
regNumber reg = call->GetRegNumByIdx(i);
var_types type = op1->GetRegTypeByIndex(i);
regNumber reg = op1->GetRegByIndex(i);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
@ -1449,12 +1442,10 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
}
else
{
// Stack store
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = pRetTypeDesc->GetReturnRegType(i);
regNumber reg = call->GetRegNumByIdx(i);
var_types type = op1->GetRegTypeByIndex(i);
regNumber reg = op1->GetRegByIndex(i);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
@ -1471,7 +1462,7 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
offset += genTypeSize(type);
}
// Updating variable liveness after instruction was emitted
// Update variable liveness.
genUpdateLife(treeNode);
varDsc->SetRegNum(REG_STK);
}
@ -2333,132 +2324,6 @@ void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode)
genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN);
}
//------------------------------------------------------------------------
// genRegCopy: Produce code for a GT_COPY node.
//
// Arguments:
// tree - the GT_COPY node
//
// Notes:
// This will copy the register(s) produced by this node's source, to
// the register(s) allocated to this GT_COPY node.
// It has some special handling for these cases:
// - when the source and target registers are in different register files
// (note that this is *not* a conversion).
// - when the source is a lclVar whose home location is being moved to a new
// register (rather than just being copied for temporary use).
//
void CodeGen::genRegCopy(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_COPY);
GenTree* op1 = treeNode->AsOp()->gtOp1;
regNumber sourceReg = genConsumeReg(op1);
if (op1->IsMultiRegNode())
{
noway_assert(!op1->IsCopyOrReload());
unsigned regCount = op1->GetMultiRegCount();
for (unsigned i = 0; i < regCount; i++)
{
regNumber srcReg = op1->GetRegByIndex(i);
regNumber tgtReg = treeNode->AsCopyOrReload()->GetRegNumByIdx(i);
var_types regType = op1->GetRegTypeByIndex(i);
inst_RV_RV(ins_Copy(regType), tgtReg, srcReg, regType);
}
}
else
{
var_types targetType = treeNode->TypeGet();
regNumber targetReg = treeNode->GetRegNum();
assert(targetReg != REG_NA);
assert(targetType != TYP_STRUCT);
// Check whether this node and the node from which we're copying the value have the same
// register type.
// This can happen if (currently iff) we have a SIMD vector type that fits in an integer
// register, in which case it is passed as an argument, or returned from a call,
// in an integer register and must be copied if it's in a floating point register.
bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1));
bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode));
if (srcFltReg != tgtFltReg)
{
#ifdef TARGET_ARM64
inst_RV_RV(INS_fmov, targetReg, sourceReg, targetType);
#else // !TARGET_ARM64
if (varTypeIsFloating(treeNode))
{
// GT_COPY from 'int' to 'float' currently can't happen. Maybe if ARM SIMD is implemented
// it will happen, according to the comment above?
NYI_ARM("genRegCopy from 'int' to 'float'");
}
else
{
assert(varTypeIsFloating(op1));
if (op1->TypeGet() == TYP_FLOAT)
{
inst_RV_RV(INS_vmov_f2i, targetReg, genConsumeReg(op1), targetType);
}
else
{
regNumber otherReg = (regNumber)treeNode->AsCopyOrReload()->gtOtherRegs[0];
assert(otherReg != REG_NA);
inst_RV_RV_RV(INS_vmov_d2i, targetReg, otherReg, genConsumeReg(op1), EA_8BYTE);
}
}
#endif // !TARGET_ARM64
}
else
{
inst_RV_RV(ins_Copy(targetType), targetReg, sourceReg, targetType);
}
}
if (op1->IsLocal())
{
// The lclVar will never be a def.
// If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will
// appropriately set the gcInfo for the copied value.
// If not, there are two cases we need to handle:
// - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable
// will remain live in its original register.
// genProduceReg() will appropriately set the gcInfo for the copied value,
// and genConsumeReg will reset it.
// - Otherwise, we need to update register info for the lclVar.
GenTreeLclVarCommon* lcl = op1->AsLclVarCommon();
assert((lcl->gtFlags & GTF_VAR_DEF) == 0);
if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0)
{
LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()];
// If we didn't just spill it (in genConsumeReg, above), then update the register info
if (varDsc->GetRegNum() != REG_STK)
{
// The old location is dying
genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1));
gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum()));
genUpdateVarReg(varDsc, treeNode);
#ifdef USING_VARIABLE_LIVE_RANGE
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum())
#endif // USING_VARIABLE_LIVE_RANGE
// The new location is going live
genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode));
}
}
}
genProduceReg(treeNode);
}
//------------------------------------------------------------------------
// genCallInstruction: Produce code for a GT_CALL node
//
@ -3759,95 +3624,28 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea)
genProduceReg(lea);
}
#ifdef FEATURE_SIMD
//------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct.
// genSIMDSplitReturn: Generates code for returning a fixed-size SIMD type that lives
// in a single register, but is returned in multiple registers.
//
// Arguments:
// treeNode - The tree node to evaluate whether is a struct return.
// src - The source of the return
// retTypeDesc - The return type descriptor.
//
// Return Value:
// Returns true if the 'treeNode" is a GT_RETURN node of type struct.
// Otherwise returns false.
//
bool CodeGen::isStructReturn(GenTree* treeNode)
void CodeGen::genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc)
{
// This method could be called for 'treeNode' of GT_RET_FILT or GT_RETURN.
// For the GT_RET_FILT, the return is always
// a bool or a void, for the end of a finally block.
noway_assert(treeNode->OperGet() == GT_RETURN || treeNode->OperGet() == GT_RETFILT);
var_types returnType = treeNode->TypeGet();
#ifdef TARGET_ARM64
return varTypeIsStruct(returnType) && (compiler->info.compRetNativeType == TYP_STRUCT);
#else
return varTypeIsStruct(returnType);
#endif
}
//------------------------------------------------------------------------
// genStructReturn: Generates code for returning a struct.
//
// Arguments:
// treeNode - The GT_RETURN tree node.
//
// Return Value:
// None
//
// Assumption:
// op1 of GT_RETURN node is either GT_LCL_VAR or multi-reg GT_CALL
void CodeGen::genStructReturn(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_RETURN);
assert(isStructReturn(treeNode));
GenTree* op1 = treeNode->gtGetOp1();
if (op1->OperGet() == GT_LCL_VAR)
{
GenTreeLclVarCommon* lclVar = op1->AsLclVarCommon();
LclVarDsc* varDsc = &(compiler->lvaTable[lclVar->GetLclNum()]);
var_types lclType = genActualType(varDsc->TypeGet());
assert(varTypeIsStruct(lclType));
assert(varDsc->lvIsMultiRegRet);
ReturnTypeDesc retTypeDesc;
unsigned regCount;
retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle());
regCount = retTypeDesc.GetReturnRegCount();
assert(regCount >= 2);
assert(varTypeIsSIMD(lclType) || op1->isContained());
if (op1->isContained())
{
// Copy var on stack into ABI return registers
// TODO: It could be optimized by reducing two float loading to one double
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber reg = retTypeDesc.GetABIReturnReg(i);
GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), reg, lclVar->GetLclNum(), offset);
offset += genTypeSize(type);
}
}
else
{
// Handle SIMD genStructReturn case
NYI_ARM("SIMD genStructReturn");
#ifdef TARGET_ARM64
genConsumeRegs(op1);
regNumber src = op1->GetRegNum();
assert(varTypeIsSIMD(src));
assert(src->isUsedFromReg());
regNumber srcReg = src->GetRegNum();
// Treat src register as a homogenous vector with element size equal to the reg size
// Insert pieces in order
unsigned regCount = retTypeDesc->GetReturnRegCount();
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber reg = retTypeDesc.GetABIReturnReg(i);
var_types type = retTypeDesc->GetReturnRegType(i);
regNumber reg = retTypeDesc->GetABIReturnReg(i);
if (varTypeIsFloating(type))
{
// If the register piece is to be passed in a floating point register
@ -3857,7 +3655,7 @@ void CodeGen::genStructReturn(GenTree* treeNode)
// This effectively moves from `src[i]` to `reg[0]`, upper bits of reg remain unchanged
// For the case where src == reg, since we are only writing reg[0], as long as we iterate
// so that src[0] is consumed before writing reg[0], we do not need a temporary.
GetEmitter()->emitIns_R_R_I_I(INS_mov, emitTypeSize(type), reg, src, 0, i);
GetEmitter()->emitIns_R_R_I_I(INS_mov, emitTypeSize(type), reg, srcReg, 0, i);
}
else
{
@ -3865,155 +3663,10 @@ void CodeGen::genStructReturn(GenTree* treeNode)
// Use a vector mov to general purpose register instruction
// mov reg, src[i]
// This effectively moves from `src[i]` to `reg`
GetEmitter()->emitIns_R_R_I(INS_mov, emitTypeSize(type), reg, src, i);
GetEmitter()->emitIns_R_R_I(INS_mov, emitTypeSize(type), reg, srcReg, i);
}
}
#endif // TARGET_ARM64
}
}
else // op1 must be multi-reg GT_CALL
{
assert(op1->IsMultiRegCall() || op1->IsCopyOrReloadOfMultiRegCall());
genConsumeRegs(op1);
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc();
const unsigned regCount = pRetTypeDesc->GetReturnRegCount();
unsigned matchingCount = 0;
var_types regType[MAX_RET_REG_COUNT];
regNumber returnReg[MAX_RET_REG_COUNT];
regNumber allocatedReg[MAX_RET_REG_COUNT];
regMaskTP srcRegsMask = 0;
regMaskTP dstRegsMask = 0;
bool needToShuffleRegs = false; // Set to true if we have to move any registers
for (unsigned i = 0; i < regCount; ++i)
{
regType[i] = pRetTypeDesc->GetReturnRegType(i);
returnReg[i] = pRetTypeDesc->GetABIReturnReg(i);
regNumber reloadReg = REG_NA;
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
// that need to be copied or reloaded.
reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(i);
}
if (reloadReg != REG_NA)
{
allocatedReg[i] = reloadReg;
}
else
{
allocatedReg[i] = call->GetRegNumByIdx(i);
}
if (returnReg[i] == allocatedReg[i])
{
matchingCount++;
}
else // We need to move this value
{
// We want to move the value from allocatedReg[i] into returnReg[i]
// so record these two registers in the src and dst masks
//
srcRegsMask |= genRegMask(allocatedReg[i]);
dstRegsMask |= genRegMask(returnReg[i]);
needToShuffleRegs = true;
}
}
if (needToShuffleRegs)
{
assert(matchingCount < regCount);
unsigned remainingRegCount = regCount - matchingCount;
regMaskTP extraRegMask = treeNode->gtRsvdRegs;
while (remainingRegCount > 0)
{
// set 'available' to the 'dst' registers that are not currently holding 'src' registers
//
regMaskTP availableMask = dstRegsMask & ~srcRegsMask;
regMaskTP dstMask;
regNumber srcReg;
regNumber dstReg;
var_types curType = TYP_UNKNOWN;
regNumber freeUpReg = REG_NA;
if (availableMask == 0)
{
// Circular register dependencies
// So just free up the lowest register in dstRegsMask by moving it to the 'extra' register
assert(dstRegsMask == srcRegsMask); // this has to be true for us to reach here
assert(extraRegMask != 0); // we require an 'extra' register
assert((extraRegMask & ~dstRegsMask) != 0); // it can't be part of dstRegsMask
availableMask = extraRegMask & ~dstRegsMask;
regMaskTP srcMask = genFindLowestBit(srcRegsMask);
freeUpReg = genRegNumFromMask(srcMask);
}
dstMask = genFindLowestBit(availableMask);
dstReg = genRegNumFromMask(dstMask);
srcReg = REG_NA;
if (freeUpReg != REG_NA)
{
// We will free up the srcReg by moving it to dstReg which is an extra register
//
srcReg = freeUpReg;
// Find the 'srcReg' and set 'curType', change allocatedReg[] to dstReg
// and add the new register mask bit to srcRegsMask
//
for (unsigned i = 0; i < regCount; ++i)
{
if (allocatedReg[i] == srcReg)
{
curType = regType[i];
allocatedReg[i] = dstReg;
srcRegsMask |= genRegMask(dstReg);
}
}
}
else // The normal case
{
// Find the 'srcReg' and set 'curType'
//
for (unsigned i = 0; i < regCount; ++i)
{
if (returnReg[i] == dstReg)
{
srcReg = allocatedReg[i];
curType = regType[i];
}
}
// After we perform this move we will have one less registers to setup
remainingRegCount--;
}
assert(curType != TYP_UNKNOWN);
inst_RV_RV(ins_Copy(curType), dstReg, srcReg, curType);
// Clear the appropriate bits in srcRegsMask and dstRegsMask
srcRegsMask &= ~genRegMask(srcReg);
dstRegsMask &= ~genRegMask(dstReg);
} // while (remainingRegCount > 0)
} // (needToShuffleRegs)
} // op1 must be multi-reg GT_CALL
}
#endif // FEATURE_SIMD
#endif // TARGET_ARMARCH

276
src/coreclr/src/jit/codegencommon.cpp
View file

@ -11627,6 +11627,282 @@ void CodeGen::genReturn(GenTree* treeNode)
#endif // defined(DEBUG) && defined(TARGET_XARCH)
}
//------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct.
//
// Arguments:
// treeNode - The tree node to evaluate whether is a struct return.
//
// Return Value:
// Returns true if the 'treeNode" is a GT_RETURN node of type struct.
// Otherwise returns false.
//
bool CodeGen::isStructReturn(GenTree* treeNode)
{
// This method could be called for 'treeNode' of GT_RET_FILT or GT_RETURN.
// For the GT_RET_FILT, the return is always a bool or a void, for the end of a finally block.
noway_assert(treeNode->OperGet() == GT_RETURN || treeNode->OperGet() == GT_RETFILT);
if (treeNode->OperGet() != GT_RETURN)
{
return false;
}
#if defined(TARGET_AMD64) && !defined(UNIX_AMD64_ABI)
assert(!varTypeIsStruct(treeNode));
return false;
#elif defined(TARGET_ARM64)
return varTypeIsStruct(treeNode) && (compiler->info.compRetNativeType == TYP_STRUCT);
#else
return varTypeIsStruct(treeNode);
#endif
}
//------------------------------------------------------------------------
// genStructReturn: Generates code for returning a struct.
//
// Arguments:
// treeNode - The GT_RETURN tree node.
//
// Return Value:
// None
//
// Assumption:
// op1 of GT_RETURN node is either GT_LCL_VAR or multi-reg GT_CALL
//
void CodeGen::genStructReturn(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_RETURN);
GenTree* op1 = treeNode->gtGetOp1();
genConsumeRegs(op1);
GenTree* actualOp1 = op1;
if (op1->IsCopyOrReload())
{
actualOp1 = op1->gtGetOp1();
}
ReturnTypeDesc retTypeDesc;
LclVarDsc* varDsc = nullptr;
if (actualOp1->OperIs(GT_LCL_VAR))
{
varDsc = compiler->lvaGetDesc(actualOp1->AsLclVar()->GetLclNum());
retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle());
}
else
{
assert(actualOp1->OperIs(GT_CALL));
retTypeDesc = *(actualOp1->AsCall()->GetReturnTypeDesc());
}
unsigned regCount = retTypeDesc.GetReturnRegCount();
assert(regCount <= MAX_RET_REG_COUNT);
#if FEATURE_MULTIREG_RET
if (actualOp1->OperIs(GT_LCL_VAR) && (varTypeIsEnregisterable(op1)))
{
// Right now the only enregisterable structs supported are SIMD vector types.
assert(varTypeIsSIMD(op1));
#ifdef FEATURE_SIMD
genSIMDSplitReturn(op1, &retTypeDesc);
#endif // FEATURE_SIMD
}
else if (actualOp1->OperIs(GT_LCL_VAR))
{
GenTreeLclVar* lclNode = actualOp1->AsLclVar();
LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum());
assert(varDsc->lvIsMultiRegRet);
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber toReg = retTypeDesc.GetABIReturnReg(i);
GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), toReg, lclNode->GetLclNum(), offset);
offset += genTypeSize(type);
}
}
else
{
assert(actualOp1->IsMultiRegCall());
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber toReg = retTypeDesc.GetABIReturnReg(i);
regNumber fromReg = op1->GetRegByIndex(i);
if (fromReg == REG_NA)
{
assert(op1->IsCopyOrReload());
fromReg = actualOp1->GetRegByIndex(i);
}
if (fromReg != toReg)
{
inst_RV_RV(ins_Copy(type), toReg, fromReg, type);
}
}
}
#else // !FEATURE_MULTIREG_RET
unreached();
#endif
}
//------------------------------------------------------------------------
// genRegCopy: Produce code for a GT_COPY node.
//
// Arguments:
// tree - the GT_COPY node
//
// Notes:
// This will copy the register(s) produced by this nodes source, to
// the register(s) allocated to this GT_COPY node.
// It has some special handling for these casess:
// - when the source and target registers are in different register files
// (note that this is *not* a conversion).
// - when the source is a lclVar whose home location is being moved to a new
// register (rather than just being copied for temporary use).
//
void CodeGen::genRegCopy(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_COPY);
GenTree* op1 = treeNode->AsOp()->gtOp1;
if (op1->IsMultiRegNode())
{
// Register allocation assumes that any reload and copy are done in operand order.
// That is, we can have:
// (reg0, reg1) = COPY(V0,V1) where V0 is in reg1 and V1 is in memory
// The register allocation model assumes:
// First, V0 is moved to reg0 (v1 can't be in reg0 because it is still live, which would be a conflict).
// Then, V1 is moved to reg1
// However, if we call genConsumeRegs on op1, it will do the reload of V1 before we do the copy of V0.
// So we need to handle that case first.
//
// There should never be any circular dependencies, and we will check that here.
GenTreeCopyOrReload* copyNode = treeNode->AsCopyOrReload();
unsigned regCount = copyNode->GetRegCount();
// GenTreeCopyOrReload only reports the number of registers that are valid.
assert(regCount <= 2);
// First set the source registers as busy if they haven't been spilled.
// (Note that this is just for verification that we don't have circular dependencies.)
regMaskTP busyRegs = RBM_NONE;
for (unsigned i = 0; i < regCount; ++i)
{
if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0)
{
busyRegs |= genRegMask(op1->GetRegByIndex(i));
}
}
// First do any copies - we'll do the reloads after all the copies are complete.
for (unsigned i = 0; i < regCount; ++i)
{
regNumber sourceReg = op1->GetRegByIndex(i);
regNumber targetReg = copyNode->GetRegNumByIdx(i);
regMaskTP targetRegMask = genRegMask(targetReg);
// GenTreeCopyOrReload only reports the number of registers that are valid.
if (targetReg != REG_NA)
{
// We shouldn't specify a no-op move.
assert(sourceReg != targetReg);
assert((busyRegs & targetRegMask) == 0);
// Clear sourceReg from the busyRegs, and add targetReg.
busyRegs &= ~genRegMask(sourceReg);
busyRegs |= genRegMask(targetReg);
var_types type;
if (op1->IsMultiRegLclVar())
{
type = op1->AsLclVar()->GetFieldTypeByIndex(compiler, i);
}
else
{
type = op1->GetRegTypeByIndex(i);
}
inst_RV_RV(ins_Copy(type), targetReg, sourceReg, type);
}
}
// Now we can consume op1, which will perform any necessary reloads.
genConsumeReg(op1);
}
else
{
var_types targetType = treeNode->TypeGet();
regNumber targetReg = treeNode->GetRegNum();
assert(targetReg != REG_NA);
assert(targetType != TYP_STRUCT);
// Check whether this node and the node from which we're copying the value have
// different register types. This can happen if (currently iff) we have a SIMD
// vector type that fits in an integer register, in which case it is passed as
// an argument, or returned from a call, in an integer register and must be
// copied if it's in an xmm register.
bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1));
bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode));
if (srcFltReg != tgtFltReg)
{
instruction ins;
regNumber fpReg;
regNumber intReg;
if (tgtFltReg)
{
ins = ins_CopyIntToFloat(op1->TypeGet(), treeNode->TypeGet());
fpReg = targetReg;
intReg = op1->GetRegNum();
}
else
{
ins = ins_CopyFloatToInt(op1->TypeGet(), treeNode->TypeGet());
intReg = targetReg;
fpReg = op1->GetRegNum();
}
inst_RV_RV(ins, fpReg, intReg, targetType);
}
else
{
inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType);
}
if (op1->IsLocal())
{
// The lclVar will never be a def.
// If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will
// appropriately set the gcInfo for the copied value.
// If not, there are two cases we need to handle:
// - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable
// will remain live in its original register.
// genProduceReg() will appropriately set the gcInfo for the copied value,
// and genConsumeReg will reset it.
// - Otherwise, we need to update register info for the lclVar.
GenTreeLclVarCommon* lcl = op1->AsLclVarCommon();
assert((lcl->gtFlags & GTF_VAR_DEF) == 0);
if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0)
{
LclVarDsc* varDsc = compiler->lvaGetDesc(lcl);
// If we didn't just spill it (in genConsumeReg, above), then update the register info
if (varDsc->GetRegNum() != REG_STK)
{
// The old location is dying
genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1));
gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum()));
genUpdateVarReg(varDsc, treeNode);
#ifdef USING_VARIABLE_LIVE_RANGE
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum());
#endif // USING_VARIABLE_LIVE_RANGE
// The new location is going live
genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode));
}
}
}
}
genProduceReg(treeNode);
}
#if defined(DEBUG) && defined(TARGET_XARCH)
//------------------------------------------------------------------------

231
src/coreclr/src/jit/codegenlinear.cpp
View file

@ -938,6 +938,86 @@ GenTree* sameRegAsDst(GenTree* tree, GenTree*& other /*out*/)
}
}
//------------------------------------------------------------------------
// genUnspillLocal: Reload a register candidate local into a register.
//
// Arguments:
// varNum - The variable number of the local to be reloaded (unspilled).
// It may be a local field.
// type - The type of the local.
// lclNode - The node being unspilled. Note that for a multi-reg local,
// the gtLclNum will be that of the parent struct.
// regNum - The register that 'varNum' should be loaded to.
// reSpill - True if it will be immediately spilled after use.
// isLastUse - True if this is a last use of 'varNum'.
//
// Notes:
// The caller must have determined that this local needs to be unspilled.
void CodeGen::genUnspillLocal(
unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse)
{
LclVarDsc* varDsc = compiler->lvaGetDesc(varNum);
inst_set_SV_var(lclNode);
instruction ins = ins_Load(type, compiler->isSIMDTypeLocalAligned(varNum));
GetEmitter()->emitIns_R_S(ins, emitTypeSize(type), regNum, varNum, 0);
// TODO-Review: We would like to call:
// genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(tree));
// instead of the following code, but this ends up hitting this assert:
// assert((regSet.GetMaskVars() & regMask) == 0);
// due to issues with LSRA resolution moves.
// So, just force it for now. This probably indicates a condition that creates a GC hole!
//
// Extra note: I think we really want to call something like gcInfo.gcUpdateForRegVarMove,
// because the variable is not really going live or dead, but that method is somewhat poorly
// factored because it, in turn, updates rsMaskVars which is part of RegSet not GCInfo.
// TODO-Cleanup: This code exists in other CodeGen*.cpp files, and should be moved to CodeGenCommon.cpp.
// Don't update the variable's location if we are just re-spilling it again.
if (!reSpill)
{
varDsc->SetRegNum(regNum);
#ifdef USING_VARIABLE_LIVE_RANGE
// We want "VariableLiveRange" inclusive on the beginning and exclusive on the ending.
// For that we shouldn't report an update of the variable location if is becoming dead
// on the same native offset.
if (!isLastUse)
{
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, varNum);
}
#endif // USING_VARIABLE_LIVE_RANGE
if (!varDsc->lvLiveInOutOfHndlr)
{
#ifdef DEBUG
if (VarSetOps::IsMember(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex))
{
JITDUMP("\t\t\t\t\t\t\tRemoving V%02u from gcVarPtrSetCur\n", varNum);
}
#endif // DEBUG
VarSetOps::RemoveElemD(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex);
}
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tV%02u in reg ", varNum);
varDsc->PrintVarReg();
printf(" is becoming live ");
compiler->printTreeID(lclNode);
printf("\n");
}
#endif // DEBUG
regSet.AddMaskVars(genGetRegMask(varDsc));
}
gcInfo.gcMarkRegPtrVal(regNum, type);
}
//------------------------------------------------------------------------
// genUnspillRegIfNeeded: Reload the value into a register, if needed
//
@ -968,8 +1048,9 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree)
// Reset spilled flag, since we are going to load a local variable from its home location.
unspillTree->gtFlags &= ~GTF_SPILLED;
GenTreeLclVarCommon* lcl = unspillTree->AsLclVarCommon();
LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()];
GenTreeLclVar* lcl = unspillTree->AsLclVar();
LclVarDsc* varDsc = compiler->lvaGetDesc(lcl->GetLclNum());
var_types spillType = unspillTree->TypeGet();
// TODO-Cleanup: The following code could probably be further merged and cleaned up.
#ifdef TARGET_XARCH
@ -985,99 +1066,26 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree)
// In the normalizeOnLoad case ins_Load will return an appropriate sign- or zero-
// extending load.
var_types treeType = unspillTree->TypeGet();
if (treeType != genActualType(varDsc->lvType) && !varTypeIsGC(treeType) && !varDsc->lvNormalizeOnLoad())
if (spillType != genActualType(varDsc->lvType) && !varTypeIsGC(spillType) && !varDsc->lvNormalizeOnLoad())
{
assert(!varTypeIsGC(varDsc));
var_types spillType = genActualType(varDsc->lvType);
unspillTree->gtType = spillType;
inst_RV_TT(ins_Load(spillType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum())), dstReg,
unspillTree);
unspillTree->gtType = treeType;
}
else
{
inst_RV_TT(ins_Load(treeType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum())), dstReg, unspillTree);
spillType = genActualType(varDsc->lvType);
}
#elif defined(TARGET_ARM64)
var_types targetType = unspillTree->gtType;
if (targetType != genActualType(varDsc->lvType) && !varTypeIsGC(targetType) && !varDsc->lvNormalizeOnLoad())
if (spillType != genActualType(varDsc->lvType) && !varTypeIsGC(spillType) && !varDsc->lvNormalizeOnLoad())
{
assert(!varTypeIsGC(varDsc));
targetType = genActualType(varDsc->lvType);
spillType = genActualType(varDsc->lvType);
}
instruction ins = ins_Load(targetType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum()));
emitAttr attr = emitActualTypeSize(targetType);
emitter* emit = GetEmitter();
// Load local variable from its home location.
inst_RV_TT(ins, dstReg, unspillTree, 0, attr);
#elif defined(TARGET_ARM)
var_types targetType = unspillTree->gtType;
instruction ins = ins_Load(targetType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum()));
emitAttr attr = emitTypeSize(targetType);
// Load local variable from its home location.
inst_RV_TT(ins, dstReg, unspillTree, 0, attr);
// No normalizing for ARM
#else
NYI("Unspilling not implemented for this target architecture.");
#endif
// TODO-Review: We would like to call:
// genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(tree));
// instead of the following code, but this ends up hitting this assert:
// assert((regSet.GetMaskVars() & regMask) == 0);
// due to issues with LSRA resolution moves.
// So, just force it for now. This probably indicates a condition that creates a GC hole!
//
// Extra note: I think we really want to call something like gcInfo.gcUpdateForRegVarMove,
// because the variable is not really going live or dead, but that method is somewhat poorly
// factored because it, in turn, updates rsMaskVars which is part of RegSet not GCInfo.
// TODO-Cleanup: This code exists in other CodeGen*.cpp files, and should be moved to CodeGenCommon.cpp.
// Don't update the variable's location if we are just re-spilling it again.
if ((unspillTree->gtFlags & GTF_SPILL) == 0)
{
genUpdateVarReg(varDsc, tree);
#ifdef USING_VARIABLE_LIVE_RANGE
// We want "VariableLiveRange" inclusive on the beginbing and exclusive on the ending.
// For that we shouldn't report an update of the variable location if is becoming dead
// on the same native offset.
if ((unspillTree->gtFlags & GTF_VAR_DEATH) == 0)
{
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum());
}
#endif // USING_VARIABLE_LIVE_RANGE
if (!varDsc->lvLiveInOutOfHndlr)
{
#ifdef DEBUG
if (VarSetOps::IsMember(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex))
{
JITDUMP("\t\t\t\t\t\t\tRemoving V%02u from gcVarPtrSetCur\n", lcl->GetLclNum());
}
#endif // DEBUG
VarSetOps::RemoveElemD(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex);
}
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tV%02u in reg ", lcl->GetLclNum());
varDsc->PrintVarReg();
printf(" is becoming live ");
compiler->printTreeID(unspillTree);
printf("\n");
}
#endif // DEBUG
regSet.AddMaskVars(genGetRegMask(varDsc));
}
gcInfo.gcMarkRegPtrVal(dstReg, unspillTree->TypeGet());
bool reSpill = ((unspillTree->gtFlags & GTF_SPILL) != 0);
bool isLastUse = lcl->IsLastUse(0);
genUnspillLocal(lcl->GetLclNum(), spillType, lcl, dstReg, reSpill, isLastUse);
}
else if (unspillTree->IsMultiRegCall())
{
@ -1849,6 +1857,41 @@ void CodeGen::genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber
genSetBlockSize(blkNode, sizeReg);
}
//-------------------------------------------------------------------------
// genSpillLocal: Generate the actual spill of a local var.
//
// Arguments:
// varNum - The variable number of the local to be spilled.
// It may be a local field.
// type - The type of the local.
// lclNode - The node being spilled. Note that for a multi-reg local,
// the gtLclNum will be that of the parent struct.
// regNum - The register that 'varNum' is currently in.
//
// Return Value:
// None.
//
void CodeGen::genSpillLocal(unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum)
{
LclVarDsc* varDsc = compiler->lvaGetDesc(varNum);
assert(!varDsc->lvNormalizeOnStore() || (type == genActualType(varDsc->TypeGet())));
// We have a register candidate local that is marked with GTF_SPILL.
// This flag generally means that we need to spill this local.
// The exception is the case of a use of an EH var use that is being "spilled"
// to the stack, indicated by GTF_SPILL (note that all EH lclVar defs are always
// spilled, i.e. write-thru).
// An EH var use is always valid on the stack (so we don't need to actually spill it),
// but the GTF_SPILL flag records the fact that the register value is going dead.
if (((lclNode->gtFlags & GTF_VAR_DEF) != 0) || !varDsc->lvLiveInOutOfHndlr)
{
// Store local variable to its home location.
// Ensure that lclVar stores are typed correctly.
GetEmitter()->emitIns_S_R(ins_Store(type, compiler->isSIMDTypeLocalAligned(varNum)), emitTypeSize(type), regNum,
varNum, 0);
}
}
//-------------------------------------------------------------------------
// genProduceReg: do liveness update for register produced by the current
// node in codegen after code has been emitted for it.
@ -1878,23 +1921,7 @@ void CodeGen::genProduceReg(GenTree* tree)
if (genIsRegCandidateLocal(tree))
{
unsigned varNum = tree->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = compiler->lvaGetDesc(varNum);
assert(!varDsc->lvNormalizeOnStore() || (tree->TypeGet() == genActualType(varDsc->TypeGet())));
// If we reach here, we have a register candidate local that is marked with GTF_SPILL.
// This flag generally means that we need to spill this local.
// The exception is the case of a use of an EH var use that is being "spilled"
// to the stack, indicated by GTF_SPILL (note that all EH lclVar defs are always
// spilled, i.e. write-thru).
// An EH var use is always valid on the stack (so we don't need to actually spill it),
// but the GTF_SPILL flag records the fact that the register value is going dead.
if (((tree->gtFlags & GTF_VAR_DEF) != 0) || !varDsc->lvLiveInOutOfHndlr)
{
// Store local variable to its home location.
// Ensure that lclVar stores are typed correctly.
inst_TT_RV(ins_Store(tree->gtType, compiler->isSIMDTypeLocalAligned(varNum)),
emitTypeSize(tree->TypeGet()), tree, tree->GetRegNum());
}
genSpillLocal(varNum, tree->TypeGet(), tree->AsLclVar(), tree->GetRegNum());
}
else
{
@ -1983,8 +2010,8 @@ void CodeGen::genProduceReg(GenTree* tree)
// the register as live, with a GC pointer, if the variable is dead.
if (!genIsRegCandidateLocal(tree) || ((tree->gtFlags & GTF_VAR_DEATH) == 0))
{
// Multi-reg call node will produce more than one register result.
// Mark all the regs produced by call node.
// Multi-reg nodes will produce more than one register result.
// Mark all the regs produced by the node.
if (tree->IsMultiRegCall())
{
const GenTreeCall* call = tree->AsCall();

431
src/coreclr/src/jit/codegenxarch.cpp
View file

@ -1124,75 +1124,25 @@ void CodeGen::genCodeForMul(GenTreeOp* treeNode)
genProduceReg(treeNode);
}
#ifdef FEATURE_SIMD
//------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct.
// genSIMDSplitReturn: Generates code for returning a fixed-size SIMD type that lives
// in a single register, but is returned in multiple registers.
//
// Arguments:
// treeNode - The tree node to evaluate whether is a struct return.
// src - The source of the return
// retTypeDesc - The return type descriptor.
//
// Return Value:
// For AMD64 *nix: returns true if the 'treeNode" is a GT_RETURN node, of type struct.
// Otherwise returns false.
// For other platforms always returns false.
//
bool CodeGen::isStructReturn(GenTree* treeNode)
void CodeGen::genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc)
{
// This method could be called for 'treeNode' of GT_RET_FILT or GT_RETURN.
// For the GT_RET_FILT, the return is always
// a bool or a void, for the end of a finally block.
noway_assert(treeNode->OperGet() == GT_RETURN || treeNode->OperGet() == GT_RETFILT);
if (treeNode->OperGet() != GT_RETURN)
{
return false;
}
#ifdef UNIX_AMD64_ABI
return varTypeIsStruct(treeNode);
#else // !UNIX_AMD64_ABI
assert(!varTypeIsStruct(treeNode));
return false;
#endif // UNIX_AMD64_ABI
}
//------------------------------------------------------------------------
// genStructReturn: Generates code for returning a struct.
//
// Arguments:
// treeNode - The GT_RETURN tree node.
//
// Return Value:
// None
//
// Assumption:
// op1 of GT_RETURN node is either GT_LCL_VAR or multi-reg GT_CALL
void CodeGen::genStructReturn(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_RETURN);
GenTree* op1 = treeNode->gtGetOp1();
#ifdef UNIX_AMD64_ABI
if (op1->OperGet() == GT_LCL_VAR)
{
GenTreeLclVarCommon* lclVar = op1->AsLclVarCommon();
LclVarDsc* varDsc = &(compiler->lvaTable[lclVar->GetLclNum()]);
assert(varDsc->lvIsMultiRegRet);
ReturnTypeDesc retTypeDesc;
retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle());
const unsigned regCount = retTypeDesc.GetReturnRegCount();
assert(regCount == MAX_RET_REG_COUNT);
if (varTypeIsEnregisterable(op1))
{
// Right now the only enregisterable structs supported are SIMD vector types.
assert(varTypeIsSIMD(op1));
assert(op1->isUsedFromReg());
assert(varTypeIsSIMD(src));
assert(src->isUsedFromReg());
// This is a case of operand is in a single reg and needs to be
// returned in multiple ABI return registers.
regNumber opReg = genConsumeReg(op1);
regNumber reg0 = retTypeDesc.GetABIReturnReg(0);
regNumber reg1 = retTypeDesc.GetABIReturnReg(1);
regNumber opReg = src->GetRegNum();
regNumber reg0 = retTypeDesc->GetABIReturnReg(0);
regNumber reg1 = retTypeDesc->GetABIReturnReg(1);
if (opReg != reg0 && opReg != reg1)
{
@ -1221,117 +1171,8 @@ void CodeGen::genStructReturn(GenTree* treeNode)
inst_RV_RV(ins_Copy(TYP_DOUBLE), reg0, opReg, TYP_DOUBLE);
}
inst_RV_RV_IV(INS_shufpd, EA_16BYTE, reg1, reg1, 0x01);
}
else
{
assert(op1->isUsedFromMemory());
// Copy var on stack into ABI return registers
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber reg = retTypeDesc.GetABIReturnReg(i);
GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), reg, lclVar->GetLclNum(), offset);
offset += genTypeSize(type);
}
}
}
else
{
assert(op1->IsMultiRegCall() || op1->IsCopyOrReloadOfMultiRegCall());
genConsumeRegs(op1);
const GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
const GenTreeCall* call = actualOp1->AsCall();
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
const unsigned regCount = retTypeDesc->GetReturnRegCount();
assert(regCount == MAX_RET_REG_COUNT);
// Handle circular dependency between call allocated regs and ABI return regs.
//
// It is possible under LSRA stress that originally allocated regs of call node,
// say rax and rdx, are spilled and reloaded to rdx and rax respectively. But
// GT_RETURN needs to move values as follows: rdx->rax, rax->rdx. Similar kind
// kind of circular dependency could arise between xmm0 and xmm1 return regs.
// Codegen is expected to handle such circular dependency.
//
var_types regType0 = retTypeDesc->GetReturnRegType(0);
regNumber returnReg0 = retTypeDesc->GetABIReturnReg(0);
regNumber allocatedReg0 = call->GetRegNumByIdx(0);
var_types regType1 = retTypeDesc->GetReturnRegType(1);
regNumber returnReg1 = retTypeDesc->GetABIReturnReg(1);
regNumber allocatedReg1 = call->GetRegNumByIdx(1);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
// that need to be copied or reloaded.
regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(0);
if (reloadReg != REG_NA)
{
allocatedReg0 = reloadReg;
}
reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(1);
if (reloadReg != REG_NA)
{
allocatedReg1 = reloadReg;
}
}
if (allocatedReg0 == returnReg1 && allocatedReg1 == returnReg0)
{
// Circular dependency - swap allocatedReg0 and allocatedReg1
if (varTypeIsFloating(regType0))
{
assert(varTypeIsFloating(regType1));
// The fastest way to swap two XMM regs is using PXOR
inst_RV_RV(INS_pxor, allocatedReg0, allocatedReg1, TYP_DOUBLE);
inst_RV_RV(INS_pxor, allocatedReg1, allocatedReg0, TYP_DOUBLE);
inst_RV_RV(INS_pxor, allocatedReg0, allocatedReg1, TYP_DOUBLE);
}
else
{
assert(varTypeIsIntegral(regType0));
assert(varTypeIsIntegral(regType1));
inst_RV_RV(INS_xchg, allocatedReg1, allocatedReg0, TYP_I_IMPL);
}
}
else if (allocatedReg1 == returnReg0)
{
// Change the order of moves to correctly handle dependency.
if (allocatedReg1 != returnReg1)
{
inst_RV_RV(ins_Copy(regType1), returnReg1, allocatedReg1, regType1);
}
if (allocatedReg0 != returnReg0)
{
inst_RV_RV(ins_Copy(regType0), returnReg0, allocatedReg0, regType0);
}
}
else
{
// No circular dependency case.
if (allocatedReg0 != returnReg0)
{
inst_RV_RV(ins_Copy(regType0), returnReg0, allocatedReg0, regType0);
}
if (allocatedReg1 != returnReg1)
{
inst_RV_RV(ins_Copy(regType1), returnReg1, allocatedReg1, regType1);
}
}
}
#else
unreached();
#endif
}
#endif // FEATURE_SIMD
#if defined(TARGET_X86)
@ -2009,7 +1850,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
}
//----------------------------------------------------------------------------------
// genMultiRegCallStoreToLocal: store multi-reg return value of a call node to a local
// genMultiRegStoreToLocal: store multi-reg return value of a call node to a local
//
// Arguments:
// treeNode - Gentree of GT_STORE_LCL_VAR
@ -2017,45 +1858,52 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
// Return Value:
// None
//
// Assumption:
// The child of store is a multi-reg call node.
// genProduceReg() on treeNode is made by caller of this routine.
// Assumptions:
// The child of store is a multi-reg node.
//
void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
void CodeGen::genMultiRegStoreToLocal(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_STORE_LCL_VAR);
#ifdef UNIX_AMD64_ABI
// Structs of size >=9 and <=16 are returned in two return registers on x64 Unix.
assert(varTypeIsStruct(treeNode));
// Assumption: current x64 Unix implementation requires that a multi-reg struct
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being struct promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = &(compiler->lvaTable[lclNum]);
noway_assert(varDsc->lvIsMultiRegRet);
assert(varTypeIsStruct(treeNode) || varTypeIsMultiReg(treeNode));
GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
assert(call->HasMultiRegRetVal());
assert(op1->IsMultiRegNode());
unsigned regCount = op1->GetMultiRegCount();
// Assumption: The current implementation requires that a multi-reg
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being struct promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum);
if (op1->OperIs(GT_CALL))
{
assert(regCount == MAX_RET_REG_COUNT);
noway_assert(varDsc->lvIsMultiRegRet);
}
genConsumeRegs(op1);
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT);
unsigned regCount = retTypeDesc->GetReturnRegCount();
#ifdef UNIX_AMD64_ABI
// Structs of size >=9 and <=16 are returned in two return registers on x64 Unix.
if (treeNode->GetRegNum() != REG_NA)
// Handle the case of a SIMD type returned in 2 registers.
if (varTypeIsSIMD(treeNode) && (treeNode->GetRegNum() != REG_NA))
{
// Right now the only enregistrable structs supported are SIMD types.
assert(varTypeIsSIMD(treeNode));
// They are only returned in 1 or 2 registers - the 1 register case is
// handled as a regular STORE_LCL_VAR.
// This case is always a call (AsCall() will assert if it is not).
GenTreeCall* call = actualOp1->AsCall();
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT);
assert(regCount == 2);
assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(0)));
assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(1)));
// This is a case of two 8-bytes that comprise the operand is in
// two different xmm registers and needs to assembled into a single
// This is a case where the two 8-bytes that comprise the operand are in
// two different xmm registers and need to be assembled into a single
// xmm register.
regNumber targetReg = treeNode->GetRegNum();
regNumber reg0 = call->GetRegNumByIdx(0);
@ -2111,13 +1959,17 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
}
}
else
#endif // UNIX_AMD64_ABI
{
// Stack store
// This may be:
// - a call returning multiple registers
// - a HW intrinsic producing two registers to be stored into a TYP_STRUCT
//
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc->GetReturnRegType(i);
regNumber reg = call->GetRegNumByIdx(i);
var_types type = op1->GetRegTypeByIndex(i);
regNumber reg = op1->GetRegByIndex(i);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
@ -2133,57 +1985,10 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset);
offset += genTypeSize(type);
}
// Update variable liveness.
genUpdateLife(treeNode);
varDsc->SetRegNum(REG_STK);
}
#elif defined(TARGET_X86)
// Longs are returned in two return registers on x86.
assert(varTypeIsLong(treeNode));
// Assumption: current x86 implementation requires that a multi-reg long
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = &(compiler->lvaTable[lclNum]);
noway_assert(varDsc->lvIsMultiRegRet);
GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
assert(call->HasMultiRegRetVal());
genConsumeRegs(op1);
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
unsigned regCount = retTypeDesc->GetReturnRegCount();
assert(regCount == MAX_RET_REG_COUNT);
// Stack store
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc->GetReturnRegType(i);
regNumber reg = call->GetRegNumByIdx(i);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
// that need to be copied or reloaded.
regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(i);
if (reloadReg != REG_NA)
{
reg = reloadReg;
}
}
assert(reg != REG_NA);
GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset);
offset += genTypeSize(type);
}
varDsc->SetRegNum(REG_STK);
#else // !UNIX_AMD64_ABI && !TARGET_X86
assert(!"Unreached");
#endif // !UNIX_AMD64_ABI && !TARGET_X86
}
//------------------------------------------------------------------------
@ -4636,9 +4441,9 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree)
// var = call, where call returns a multi-reg return value
// case is handled separately.
if (op1->gtSkipReloadOrCopy()->IsMultiRegCall())
if (op1->gtSkipReloadOrCopy()->IsMultiRegNode())
{
genMultiRegCallStoreToLocal(tree);
genMultiRegStoreToLocal(tree);
}
else
{
@ -4874,130 +4679,6 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree)
genProduceReg(tree);
}
//------------------------------------------------------------------------
// genRegCopy: Produce code for a GT_COPY node.
//
// Arguments:
// tree - the GT_COPY node
//
// Notes:
// This will copy the register(s) produced by this nodes source, to
// the register(s) allocated to this GT_COPY node.
// It has some special handling for these casess:
// - when the source and target registers are in different register files
// (note that this is *not* a conversion).
// - when the source is a lclVar whose home location is being moved to a new
// register (rather than just being copied for temporary use).
//
void CodeGen::genRegCopy(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_COPY);
GenTree* op1 = treeNode->AsOp()->gtOp1;
if (op1->IsMultiRegNode())
{
genConsumeReg(op1);
GenTreeCopyOrReload* copyTree = treeNode->AsCopyOrReload();
unsigned regCount = treeNode->GetMultiRegCount();
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = op1->GetRegTypeByIndex(i);
regNumber fromReg = op1->GetRegByIndex(i);
regNumber toReg = copyTree->GetRegNumByIdx(i);
// A Multi-reg GT_COPY node will have a valid reg only for those positions for which a corresponding
// result reg of the multi-reg node needs to be copied.
if (toReg != REG_NA)
{
assert(toReg != fromReg);
inst_RV_RV(ins_Copy(type), toReg, fromReg, type);
}
}
}
else
{
var_types targetType = treeNode->TypeGet();
regNumber targetReg = treeNode->GetRegNum();
assert(targetReg != REG_NA);
// Check whether this node and the node from which we're copying the value have
// different register types. This can happen if (currently iff) we have a SIMD
// vector type that fits in an integer register, in which case it is passed as
// an argument, or returned from a call, in an integer register and must be
// copied if it's in an xmm register.
bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1));
bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode));
if (srcFltReg != tgtFltReg)
{
instruction ins;
regNumber fpReg;
regNumber intReg;
if (tgtFltReg)
{
ins = ins_CopyIntToFloat(op1->TypeGet(), treeNode->TypeGet());
fpReg = targetReg;
intReg = op1->GetRegNum();
}
else
{
ins = ins_CopyFloatToInt(op1->TypeGet(), treeNode->TypeGet());
intReg = targetReg;
fpReg = op1->GetRegNum();
}
inst_RV_RV(ins, fpReg, intReg, targetType);
}
else
{
inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType);
}
if (op1->IsLocal())
{
// The lclVar will never be a def.
// If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will
// appropriately set the gcInfo for the copied value.
// If not, there are two cases we need to handle:
// - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable
// will remain live in its original register.
// genProduceReg() will appropriately set the gcInfo for the copied value,
// and genConsumeReg will reset it.
// - Otherwise, we need to update register info for the lclVar.
GenTreeLclVarCommon* lcl = op1->AsLclVarCommon();
assert((lcl->gtFlags & GTF_VAR_DEF) == 0);
if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0)
{
LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()];
// If we didn't just spill it (in genConsumeReg, above), then update the register info
if (varDsc->GetRegNum() != REG_STK)
{
// The old location is dying
genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1));
gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum()));
genUpdateVarReg(varDsc, treeNode);
#ifdef USING_VARIABLE_LIVE_RANGE
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum());
#endif // USING_VARIABLE_LIVE_RANGE
// The new location is going live
genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode));
}
}
}
}
genProduceReg(treeNode);
}
//------------------------------------------------------------------------
// genCodeForStoreInd: Produce code for a GT_STOREIND node.
//

72
src/coreclr/src/jit/gentree.h
View file

@ -775,14 +775,14 @@ public:
// Note that a node marked GTF_VAR_MULTIREG can only be a pure definition of all the fields, or a pure use of all the fields,
// so we don't need the equivalent of GTF_VAR_USEASG.
#define GTF_VAR_MULTIREG_DEATH0 0x20000000 // GT_LCL_VAR -- The last-use bit for a lclVar (the first register if it is multireg).
#define GTF_VAR_MULTIREG_DEATH0 0x04000000 // GT_LCL_VAR -- The last-use bit for a lclVar (the first register if it is multireg).
#define GTF_VAR_DEATH GTF_VAR_MULTIREG_DEATH0
#define GTF_VAR_MULTIREG_DEATH1 0x10000000 // GT_LCL_VAR -- The last-use bit for the second register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH2 0x08000000 // GT_LCL_VAR -- The last-use bit for the third register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH3 0x04000000 // GT_LCL_VAR -- The last-use bit for the fourth register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH1 0x08000000 // GT_LCL_VAR -- The last-use bit for the second register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH2 0x10000000 // GT_LCL_VAR -- The last-use bit for the third register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH3 0x20000000 // GT_LCL_VAR -- The last-use bit for the fourth register of a multireg lclVar.
#define GTF_VAR_DEATH_MASK (GTF_VAR_MULTIREG_DEATH0|GTF_VAR_MULTIREG_DEATH1 | GTF_VAR_MULTIREG_DEATH2 | GTF_VAR_MULTIREG_DEATH3)
// This is the amount we have to shift, plus the regIndex, to get the last use bit we want.
#define MULTIREG_LAST_USE_SHIFT 17
// This is the amount we have to shift, plus the regIndex, to get the last use bit we want.
#define MULTIREG_LAST_USE_SHIFT 26
#define GTF_VAR_MULTIREG 0x02000000 // This is a struct or (on 32-bit platforms) long variable that is used or defined
// to/from a multireg source or destination (e.g. a call arg or return, or an op
// that returns its result in multiple registers such as a long multiply).
@ -1733,6 +1733,9 @@ public:
// Returns the type of the regIndex'th register defined by a multi-reg node.
var_types GetRegTypeByIndex(int regIndex);
// Returns the GTF flag equivalent for the regIndex'th register of a multi-reg node.
unsigned int GetRegSpillFlagByIdx(int regIndex) const;
// Returns true if it is a GT_COPY or GT_RELOAD node
inline bool IsCopyOrReload() const;
@ -3233,6 +3236,7 @@ private:
unsigned int GetLastUseBit(int regIndex)
{
assert(regIndex < 4);
static_assert_no_msg((1 << MULTIREG_LAST_USE_SHIFT) == GTF_VAR_MULTIREG_DEATH0);
return (1 << (MULTIREG_LAST_USE_SHIFT + regIndex));
}
@ -3251,6 +3255,7 @@ public:
void SetMultiReg()
{
gtFlags |= GTF_VAR_MULTIREG;
ClearOtherRegFlags();
}
regNumber GetRegNumByIdx(int regIndex)
@ -7303,6 +7308,61 @@ inline var_types GenTree::GetRegTypeByIndex(int regIndex)
return TYP_UNDEF;
}
//-----------------------------------------------------------------------------------
// GetRegSpillFlagByIdx: Get a specific register's spill flags, based on regIndex,
// for this multi-reg node.
//
// Arguments:
// regIndex - which register's spill flags to return
//
// Return Value:
// The spill flags (GTF_SPILL GTF_SPILLED) for this register.
//
// Notes:
// This must be a multireg node and 'regIndex' must be a valid index for this node.
// This method returns the GTF "equivalent" flags based on the packed flags on the multireg node.
//
inline unsigned int GenTree::GetRegSpillFlagByIdx(int regIndex) const
{
#if FEATURE_MULTIREG_RET
if (IsMultiRegCall())
{
return AsCall()->AsCall()->GetRegSpillFlagByIdx(regIndex);
}
#if FEATURE_ARG_SPLIT
if (OperIsPutArgSplit())
{
return AsPutArgSplit()->GetRegSpillFlagByIdx(regIndex);
}
#endif
#if !defined(TARGET_64BIT)
if (OperIsMultiRegOp())
{
return AsMultiRegOp()->GetRegSpillFlagByIdx(regIndex);
}
#endif
#endif // FEATURE_MULTIREG_RET
#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS)
if (OperIs(GT_HWINTRINSIC))
{
// At this time, the only multi-reg HW intrinsics all return the type of their
// arguments. If this changes, we will need a way to record or determine this.
assert(TypeGet() == TYP_STRUCT);
return gtGetOp1()->TypeGet();
}
#endif
if (OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR))
{
return AsLclVar()->GetRegSpillFlagByIdx(regIndex);
}
assert(!"Invalid node type for GetRegSpillFlagByIdx");
return TYP_UNDEF;
}
//-------------------------------------------------------------------------
// IsCopyOrReload: whether this is a GT_COPY or GT_RELOAD node.
//

7
src/coreclr/src/jit/lclvars.cpp
View file

@ -1971,10 +1971,12 @@ bool Compiler::StructPromotionHelper::ShouldPromoteStructVar(unsigned lclNum)
//
void Compiler::StructPromotionHelper::SortStructFields()
{
assert(!structPromotionInfo.fieldsSorted);
if (!structPromotionInfo.fieldsSorted)
{
qsort(structPromotionInfo.fields, structPromotionInfo.fieldCnt, sizeof(*structPromotionInfo.fields),
lvaFieldOffsetCmp);
structPromotionInfo.fieldsSorted = true;
}
}
//--------------------------------------------------------------------------------------------
@ -2158,10 +2160,7 @@ void Compiler::StructPromotionHelper::PromoteStructVar(unsigned lclNum)
}
#endif
if (!structPromotionInfo.fieldsSorted)
{
SortStructFields();
}
for (unsigned index = 0; index < structPromotionInfo.fieldCnt; ++index)
{

51
src/coreclr/src/jit/lsra.cpp
View file

@ -1791,6 +1791,8 @@ void LinearScan::identifyCandidates()
VarSetOps::AddElemD(compiler, fpMaybeCandidateVars, varDsc->lvVarIndex);
}
}
JITDUMP(" ");
DBEXEC(VERBOSE, newInt->dump());
}
else
{
@ -6282,6 +6284,26 @@ void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval, Regi
#endif
}
//-----------------------------------------------------------------------------
// writeLocalReg: Write the register assignment for a GT_LCL_VAR node.
//
// Arguments:
// lclNode - The GT_LCL_VAR node
// varNum - The variable number for the register
// reg - The assigned register
//
// Return Value:
// None
//
void LinearScan::writeLocalReg(GenTreeLclVar* lclNode, unsigned varNum, regNumber reg)
{
// We don't yet support multireg locals.
assert((lclNode->GetLclNum() == varNum) && !lclNode->IsMultiReg());
assert(lclNode->GetLclNum() == varNum);
lclNode->SetRegNum(reg);
}
//-----------------------------------------------------------------------------
// LinearScan::resolveLocalRef
// Description:
// Update the graph for a local reference.
@ -6318,7 +6340,7 @@ void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval, Regi
// NICE: Consider tracking whether an Interval is always in the same location (register/stack)
// in which case it will require no resolution.
//
void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPosition* currentRefPosition)
void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, RefPosition* currentRefPosition)
{
assert((block == nullptr) == (treeNode == nullptr));
assert(enregisterLocalVars);
@ -6339,11 +6361,11 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
{
if (currentRefPosition->lastUse)
{
treeNode->gtFlags |= GTF_VAR_DEATH;
treeNode->SetLastUse(currentRefPosition->getMultiRegIdx());
}
else
{
treeNode->gtFlags &= ~GTF_VAR_DEATH;
treeNode->ClearLastUse(currentRefPosition->getMultiRegIdx());
}
if ((currentRefPosition->registerAssignment != RBM_NONE) && (interval->physReg == REG_NA) &&
@ -6354,7 +6376,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
// during resolution. In this case we're better off making it contained.
assert(inVarToRegMaps[curBBNum][varDsc->lvVarIndex] == REG_STK);
currentRefPosition->registerAssignment = RBM_NONE;
treeNode->SetRegNum(REG_NA);
writeLocalReg(treeNode->AsLclVar(), interval->varNum, REG_NA);
}
}
@ -6445,9 +6467,11 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
//
// Note that varDsc->GetRegNum() is already to REG_STK above.
interval->physReg = REG_NA;
treeNode->SetRegNum(REG_NA);
writeLocalReg(treeNode->AsLclVar(), interval->varNum, REG_NA);
treeNode->gtFlags &= ~GTF_SPILLED;
treeNode->SetContained();
// We don't support RegOptional for multi-reg localvars.
assert(!treeNode->IsMultiReg());
}
else
{
@ -6470,7 +6494,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
interval->physReg = REG_NA;
if (treeNode != nullptr)
{
treeNode->SetRegNum(REG_NA);
writeLocalReg(treeNode->AsLclVar(), interval->varNum, REG_NA);
}
}
else // Not reload and Not pure-def that's spillAfter
@ -6489,7 +6513,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
// But for copyReg, the homeReg remains unchanged.
assert(treeNode != nullptr);
treeNode->SetRegNum(interval->physReg);
writeLocalReg(treeNode->AsLclVar(), interval->varNum, interval->physReg);
if (currentRefPosition->copyReg)
{
@ -7345,9 +7369,9 @@ void LinearScan::resolveRegisters()
{
writeRegisters(currentRefPosition, treeNode);
if (treeNode->IsLocal() && currentRefPosition->getInterval()->isLocalVar)
if (treeNode->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && currentRefPosition->getInterval()->isLocalVar)
{
resolveLocalRef(block, treeNode, currentRefPosition);
resolveLocalRef(block, treeNode->AsLclVar(), currentRefPosition);
}
// Mark spill locations on temps
@ -7370,7 +7394,7 @@ void LinearScan::resolveRegisters()
treeNode->ResetReuseRegVal();
}
// In case of multi-reg call node, also set spill flag on the
// In case of multi-reg node, also set spill flag on the
// register specified by multi-reg index of current RefPosition.
// Note that the spill flag on treeNode indicates that one or
// more its allocated registers are in that state.
@ -9305,7 +9329,7 @@ void Interval::dump()
}
if (isStructField)
{
printf(" (struct)");
printf(" (field)");
}
if (isPromotedStruct)
{
@ -9495,7 +9519,7 @@ void LinearScan::lsraGetOperandString(GenTree* tree,
void LinearScan::lsraDispNode(GenTree* tree, LsraTupleDumpMode mode, bool hasDest)
{
Compiler* compiler = JitTls::GetCompiler();
const unsigned operandStringLength = 16;
const unsigned operandStringLength = 6 * MAX_MULTIREG_COUNT + 1;
char operandString[operandStringLength];
const char* emptyDestOperand = " ";
char spillChar = ' ';
@ -9635,7 +9659,7 @@ void LinearScan::TupleStyleDump(LsraTupleDumpMode mode)
{
BasicBlock* block;
LsraLocation currentLoc = 1; // 0 is the entry
const unsigned operandStringLength = 16;
const unsigned operandStringLength = 6 * MAX_MULTIREG_COUNT + 1;
char operandString[operandStringLength];
// currentRefPosition is not used for LSRA_DUMP_PRE
@ -10799,6 +10823,7 @@ void LinearScan::verifyFinalAllocation()
regRecord->assignedInterval = interval;
if (VERBOSE)
{
dumpEmptyRefPosition();
printf("Move %-4s ", getRegName(regRecord->regNum));
}
}

8
src/coreclr/src/jit/lsra.h
View file

@ -965,8 +965,8 @@ private:
#ifdef DEBUG
void checkLastUses(BasicBlock* block);
static int ComputeOperandDstCount(GenTree* operand);
static int ComputeAvailableSrcCount(GenTree* node);
int ComputeOperandDstCount(GenTree* operand);
int ComputeAvailableSrcCount(GenTree* node);
#endif // DEBUG
void setFrameType();
@ -1090,7 +1090,8 @@ private:
RefPosition* buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskTP internalCands = RBM_NONE);
void buildInternalRegisterUses();
void resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPosition* currentRefPosition);
void writeLocalReg(GenTreeLclVar* lclNode, unsigned varNum, regNumber reg);
void resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, RefPosition* currentRefPosition);
void insertMove(BasicBlock* block, GenTree* insertionPoint, unsigned lclNum, regNumber inReg, regNumber outReg);
@ -1575,6 +1576,7 @@ private:
int BuildBlockStore(GenTreeBlk* blkNode);
int BuildModDiv(GenTree* tree);
int BuildIntrinsic(GenTree* tree);
void BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, LclVarDsc* varDsc, RefPosition* singleUseRef, int index);
int BuildStoreLoc(GenTreeLclVarCommon* tree);
int BuildIndir(GenTreeIndir* indirTree);
int BuildGCWriteBarrier(GenTree* tree);

2
src/coreclr/src/jit/morph.cpp
View file

@ -1950,8 +1950,6 @@ GenTree* Compiler::fgMakeTmpArgNode(fgArgTabEntry* curArgTabEntry)
assert(varTypeIsStruct(type));
if (lvaIsMultiregStruct(varDsc, curArgTabEntry->IsVararg()))
{
// ToDo-ARM64: Consider using: arg->ChangeOper(GT_LCL_FLD);
// as that is how UNIX_AMD64_ABI works.
// We will create a GT_OBJ for the argument below.
// This will be passed by value in two registers.
assert(addrNode != nullptr);

2
src/coreclr/src/jit/rationalize.cpp
View file

@ -295,7 +295,7 @@ static void RewriteAssignmentIntoStoreLclCore(GenTreeOp* assignment,
store->AsLclFld()->SetFieldSeq(var->AsLclFld()->GetFieldSeq());
}
copyFlags(store, var, GTF_LIVENESS_MASK);
copyFlags(store, var, (GTF_LIVENESS_MASK | GTF_VAR_MULTIREG));
store->gtFlags &= ~GTF_REVERSE_OPS;
store->gtType = var->TypeGet();

38
src/coreclr/src/jit/regset.cpp
View file

@ -285,17 +285,16 @@ RegSet::SpillDsc* RegSet::rsGetSpillInfo(GenTree* tree, regNumber reg, SpillDsc*
// Return Value:
// None.
//
// Assumption:
// RyuJIT backend specific: in case of multi-reg call nodes, GTF_SPILL
// flag associated with the reg that is being spilled is cleared. The
// caller of this method is expected to clear GTF_SPILL flag on call
// node after all of its registers marked for spilling are spilled.
// Notes:
// For multi-reg nodes, only the spill flag associated with this reg is cleared.
// The spill flag on the node should be cleared by the caller of this method.
//
void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
{
assert(tree != nullptr);
GenTreeCall* call = nullptr;
GenTreeLclVar* lcl = nullptr;
var_types treeType;
#if defined(TARGET_ARM)
GenTreePutArgSplit* splitArg = nullptr;
@ -320,6 +319,12 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
treeType = multiReg->GetRegType(regIdx);
}
#endif // TARGET_ARM
else if (tree->IsMultiRegLclVar())
{
GenTreeLclVar* lcl = tree->AsLclVar();
LclVarDsc* varDsc = m_rsCompiler->lvaGetDesc(lcl->GetLclNum());
treeType = varDsc->TypeGet();
}
else
{
treeType = tree->TypeGet();
@ -345,9 +350,8 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
// vars should be handled elsewhere, and to prevent
// spilling twice clear GTF_SPILL flag on tree node.
//
// In case of multi-reg call nodes only the spill flag
// associated with the reg is cleared. Spill flag on
// call node should be cleared by the caller of this method.
// In case of multi-reg nodes, only the spill flag associated with this reg is cleared.
// The spill flag on the node should be cleared by the caller of this method.
assert((tree->gtFlags & GTF_SPILL) != 0);
unsigned regFlags = 0;
@ -371,6 +375,12 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
regFlags &= ~GTF_SPILL;
}
#endif // TARGET_ARM
else if (lcl != nullptr)
{
regFlags = lcl->GetRegSpillFlagByIdx(regIdx);
assert((regFlags & GTF_SPILL) != 0);
regFlags &= ~GTF_SPILL;
}
else
{
assert(!varTypeIsMultiReg(tree));
@ -446,6 +456,11 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
multiReg->SetRegSpillFlagByIdx(regFlags, regIdx);
}
#endif // TARGET_ARM
else if (lcl != nullptr)
{
regFlags |= GTF_SPILLED;
lcl->SetRegSpillFlagByIdx(regFlags, regIdx);
}
}
#if defined(TARGET_X86)
@ -565,6 +580,13 @@ TempDsc* RegSet::rsUnspillInPlace(GenTree* tree, regNumber oldReg, unsigned regI
multiReg->SetRegSpillFlagByIdx(flags, regIdx);
}
#endif // TARGET_ARM
else if (tree->IsMultiRegLclVar())
{
GenTreeLclVar* lcl = tree->AsLclVar();
unsigned flags = lcl->GetRegSpillFlagByIdx(regIdx);
flags &= ~GTF_SPILLED;
lcl->SetRegSpillFlagByIdx(flags, regIdx);
}
else
{
tree->gtFlags &= ~GTF_SPILLED;

4
src/coreclr/src/jit/ssabuilder.cpp
View file

@ -1650,6 +1650,10 @@ bool SsaBuilder::IncludeInSsa(unsigned lclNum)
//
return false;
}
else if (varDsc->lvIsStructField && m_pCompiler->lvaGetDesc(varDsc->lvParentLcl)->lvIsMultiRegRet)
{
return false;
}
// otherwise this variable is included in SSA
return true;
}

60
src/coreclr/src/jit/treelifeupdater.cpp
View file

@ -74,9 +74,27 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
}
// if it's a partial definition then variable "x" must have had a previous, original, site to be born.
bool isBorn = ((lclVarTree->gtFlags & GTF_VAR_DEF) != 0 && (lclVarTree->gtFlags & GTF_VAR_USEASG) == 0);
bool isDying = ((lclVarTree->gtFlags & GTF_VAR_DEATH) != 0);
bool spill = ((lclVarTree->gtFlags & GTF_SPILL) != 0);
bool isBorn;
bool isDying;
bool spill;
bool isMultiRegLocal = lclVarTree->IsMultiRegLclVar();
if (isMultiRegLocal)
{
assert((tree->gtFlags & GTF_VAR_USEASG) == 0);
isBorn = ((tree->gtFlags & GTF_VAR_DEF) != 0);
// Note that for multireg locals we can have definitions for which some of those are last uses.
// We don't want to add those to the varDeltaSet because otherwise they will be added as newly
// live.
isDying = !isBorn && tree->AsLclVar()->HasLastUse();
// GTF_SPILL will be set if any registers need to be spilled.
spill = ((tree->gtFlags & GTF_SPILL) != 0);
}
else
{
isBorn = ((lclVarTree->gtFlags & GTF_VAR_DEF) != 0 && (lclVarTree->gtFlags & GTF_VAR_USEASG) == 0);
isDying = ((lclVarTree->gtFlags & GTF_VAR_DEATH) != 0);
spill = ((lclVarTree->gtFlags & GTF_SPILL) != 0);
}
// Since all tracked vars are register candidates, but not all are in registers at all times,
// we maintain two separate sets of variables - the total set of variables that are either
@ -108,6 +126,10 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
}
}
}
else if (ForCodeGen && lclVarTree->IsMultiRegLclVar())
{
assert(!"MultiRegLclVars not yet supported");
}
else if (varDsc->lvPromoted)
{
// If hasDeadTrackedFieldVars is true, then, for a LDOBJ(ADDR(<promoted struct local>)),
@ -126,15 +148,17 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
}
}
for (unsigned i = varDsc->lvFieldLclStart; i < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; ++i)
unsigned firstFieldVarNum = varDsc->lvFieldLclStart;
for (unsigned i = 0; i < varDsc->lvFieldCnt; ++i)
{
LclVarDsc* fldVarDsc = &(compiler->lvaTable[i]);
LclVarDsc* fldVarDsc = compiler->lvaGetDesc(firstFieldVarNum + i);
noway_assert(fldVarDsc->lvIsStructField);
if (fldVarDsc->lvTracked)
{
unsigned fldVarIndex = fldVarDsc->lvVarIndex;
bool isInReg = fldVarDsc->lvIsInReg();
bool isInMemory = !isInReg || fldVarDsc->lvLiveInOutOfHndlr;
// We should never see enregistered fields in a struct local unless
// IsMultiRegLclVar() returns true, in which case we've handled this above.
assert(!fldVarDsc->lvIsInReg());
noway_assert(fldVarIndex < compiler->lvaTrackedCount);
if (!hasDeadTrackedFieldVars)
{
@ -143,38 +167,16 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
{
// We repeat this call here and below to avoid the VarSetOps::IsMember
// test in this, the common case, where we have no deadTrackedFieldVars.
if (isInReg)
{
if (isBorn)
{
compiler->codeGen->genUpdateVarReg(fldVarDsc, tree);
}
compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(tree));
}
if (isInMemory)
{
VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex);
}
}
}
else if (ForCodeGen && VarSetOps::IsMember(compiler, varDeltaSet, fldVarIndex))
{
if (isInReg)
{
if (isBorn)
{
compiler->codeGen->genUpdateVarReg(fldVarDsc, tree);
}
compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(tree));
}
if (isInMemory)
{
VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex);
}
}
}
}
}
// First, update the live set
if (isDying)