VSadov/Satori
1
0
Fork 0
mirror of https://github.com/VSadov/Satori.git synced 2025-06-10 01:50:53 +09:00
Code Activity

Next round of multireg preliminary changes (#36155)

Browse source 
This is a zero-diff set of mostly refactoring changes in preparation for supporting multireg locals:
- Move `genRegCopy()` and `genStructReturn()` to codegencommon.cpp, making a new method for `genSIMDSplitReturn` which is target-specific.
- Factor out a new `genUnspillLocal` method from `genUnspillRegIfNeeded()`.
- Similarly factor out `genSpillLocal()`
- Rename `genMultiRegCallStoreToLocal()` and more generally support multireg local stores.
- Fix a bug in the order and shift amount for last-use bits on `GenTreeLclVar`
- Some additional cleanup and preparatory changes
This commit is contained in:
Carol Eidt 2020-05-15 13:04:26 -07:00 • committed by GitHub
parent b764cae5f8
commit ca0fd167d5
Signed by: github
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 731 additions and 976 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/coreclr/src/jit/codegen.h
View file

@ -1113,6 +1113,9 @@ protected:
// Do liveness update for register produced by the current node in codegen after // Do liveness update for register produced by the current node in codegen after
// code has been emitted for it. // code has been emitted for it.
void genProduceReg(GenTree* tree); void genProduceReg(GenTree* tree);
void genSpillLocal(unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum);
void genUnspillLocal(
unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse);
void genUnspillRegIfNeeded(GenTree* tree); void genUnspillRegIfNeeded(GenTree* tree);
regNumber genConsumeReg(GenTree* tree); regNumber genConsumeReg(GenTree* tree);
void genCopyRegIfNeeded(GenTree* tree, regNumber needReg); void genCopyRegIfNeeded(GenTree* tree, regNumber needReg);
@ -1275,10 +1278,13 @@ protected:
void genEHFinallyOrFilterRet(BasicBlock* block); void genEHFinallyOrFilterRet(BasicBlock* block);
#endif // !FEATURE_EH_FUNCLETS #endif // !FEATURE_EH_FUNCLETS
void genMultiRegCallStoreToLocal(GenTree* treeNode); void genMultiRegStoreToLocal(GenTree* treeNode);
// Deals with codegen for muti-register struct returns. // Codegen for multi-register struct returns.
bool isStructReturn(GenTree* treeNode); bool isStructReturn(GenTree* treeNode);
#ifdef FEATURE_SIMD
void genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc);
#endif
void genStructReturn(GenTree* treeNode); void genStructReturn(GenTree* treeNode);
#if defined(TARGET_X86) || defined(TARGET_ARM) #if defined(TARGET_X86) || defined(TARGET_ARM)

4
src/coreclr/src/jit/codegenarm.cpp
View file

@ -961,7 +961,7 @@ void CodeGen::genCodeForLclVar(GenTreeLclVar* tree)
// If this is a register candidate that has been spilled, genConsumeReg() will // If this is a register candidate that has been spilled, genConsumeReg() will
// reload it at the point of use. Otherwise, if it's not in a register, we load it here. // reload it at the point of use. Otherwise, if it's not in a register, we load it here.
if (!isRegCandidate && !(tree->gtFlags & GTF_SPILLED)) if (!isRegCandidate && !tree->IsMultiReg() && !(tree->gtFlags & GTF_SPILLED))
{ {
const LclVarDsc* varDsc = compiler->lvaGetDesc(tree); const LclVarDsc* varDsc = compiler->lvaGetDesc(tree);
var_types type = varDsc->GetRegisterType(tree); var_types type = varDsc->GetRegisterType(tree);
@ -1050,7 +1050,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree)
// case is handled separately. // case is handled separately.
if (data->gtSkipReloadOrCopy()->IsMultiRegCall()) if (data->gtSkipReloadOrCopy()->IsMultiRegCall())
{ {
genMultiRegCallStoreToLocal(tree); genMultiRegStoreToLocal(tree);
} }
else else
{ {

4
src/coreclr/src/jit/codegenarm64.cpp
View file

@ -1839,7 +1839,7 @@ void CodeGen::genCodeForLclVar(GenTreeLclVar* tree)
// If this is a register candidate that has been spilled, genConsumeReg() will // If this is a register candidate that has been spilled, genConsumeReg() will
// reload it at the point of use. Otherwise, if it's not in a register, we load it here. // reload it at the point of use. Otherwise, if it's not in a register, we load it here.
if (!isRegCandidate && !(tree->gtFlags & GTF_SPILLED)) if (!isRegCandidate && !tree->IsMultiReg() && !(tree->gtFlags & GTF_SPILLED))
{ {
// targetType must be a normal scalar type and not a TYP_STRUCT // targetType must be a normal scalar type and not a TYP_STRUCT
assert(targetType != TYP_STRUCT); assert(targetType != TYP_STRUCT);
@ -1929,7 +1929,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree)
// case is handled separately. // case is handled separately.
if (data->gtSkipReloadOrCopy()->IsMultiRegCall()) if (data->gtSkipReloadOrCopy()->IsMultiRegCall())
{ {
genMultiRegCallStoreToLocal(tree); genMultiRegStoreToLocal(tree);
} }
else else
{ {

419
src/coreclr/src/jit/codegenarmarch.cpp
View file

@ -1355,7 +1355,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode)
#endif // FEATURE_ARG_SPLIT #endif // FEATURE_ARG_SPLIT
//---------------------------------------------------------------------------------- //----------------------------------------------------------------------------------
// genMultiRegCallStoreToLocal: store multi-reg return value of a call node to a local // genMultiRegStoreToLocal: store multi-reg return value of a call node to a local
// //
// Arguments: // Arguments:
// treeNode - Gentree of GT_STORE_LCL_VAR // treeNode - Gentree of GT_STORE_LCL_VAR
@ -1364,42 +1364,35 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode)
// None // None
// //
// Assumption: // Assumption:
// The child of store is a multi-reg call node. // The child of store is a multi-reg node.
// genProduceReg() on treeNode is made by caller of this routine.
// //
void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode) void CodeGen::genMultiRegStoreToLocal(GenTree* treeNode)
{ {
assert(treeNode->OperGet() == GT_STORE_LCL_VAR); assert(treeNode->OperGet() == GT_STORE_LCL_VAR);
assert(varTypeIsStruct(treeNode) || varTypeIsMultiReg(treeNode));
#if defined(TARGET_ARM) GenTree* op1 = treeNode->gtGetOp1();
// Longs are returned in two return registers on Arm32. GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
// Structs are returned in four registers on ARM32 and HFAs. assert(op1->IsMultiRegNode());
assert(varTypeIsLong(treeNode) || varTypeIsStruct(treeNode)); unsigned regCount = op1->GetMultiRegCount();
#elif defined(TARGET_ARM64)
// Structs of size >=9 and <=16 are returned in two return registers on ARM64 and HFAs.
assert(varTypeIsStruct(treeNode));
#endif // TARGET*
// Assumption: current implementation requires that a multi-reg // Assumption: current implementation requires that a multi-reg
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from // var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being promoted. // being promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum(); unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = &(compiler->lvaTable[lclNum]); LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum);
if (op1->OperIs(GT_CALL))
{
assert(regCount <= MAX_RET_REG_COUNT);
noway_assert(varDsc->lvIsMultiRegRet); noway_assert(varDsc->lvIsMultiRegRet);
}
GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
assert(call->HasMultiRegRetVal());
genConsumeRegs(op1); genConsumeRegs(op1);
const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc(); int offset = 0;
const unsigned regCount = pRetTypeDesc->GetReturnRegCount();
if (treeNode->GetRegNum() != REG_NA) // Check for the case of an enregistered SIMD type that's returned in multiple registers.
if (varDsc->lvIsRegCandidate() && treeNode->GetRegNum() != REG_NA)
{ {
// Right now the only enregistrable multi-reg return types supported are SIMD types.
assert(varTypeIsSIMD(treeNode)); assert(varTypeIsSIMD(treeNode));
assert(regCount != 0); assert(regCount != 0);
@ -1409,8 +1402,8 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
// Insert pieces in reverse order // Insert pieces in reverse order
for (int i = regCount - 1; i >= 0; --i) for (int i = regCount - 1; i >= 0; --i)
{ {
var_types type = pRetTypeDesc->GetReturnRegType(i); var_types type = op1->GetRegTypeByIndex(i);
regNumber reg = call->GetRegNumByIdx(i); regNumber reg = op1->GetRegByIndex(i);
if (op1->IsCopyOrReload()) if (op1->IsCopyOrReload())
{ {
// GT_COPY/GT_RELOAD will have valid reg for those positions // GT_COPY/GT_RELOAD will have valid reg for those positions
@ -1449,12 +1442,10 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
} }
else else
{ {
// Stack store
int offset = 0;
for (unsigned i = 0; i < regCount; ++i) for (unsigned i = 0; i < regCount; ++i)
{ {
var_types type = pRetTypeDesc->GetReturnRegType(i); var_types type = op1->GetRegTypeByIndex(i);
regNumber reg = call->GetRegNumByIdx(i); regNumber reg = op1->GetRegByIndex(i);
if (op1->IsCopyOrReload()) if (op1->IsCopyOrReload())
{ {
// GT_COPY/GT_RELOAD will have valid reg for those positions // GT_COPY/GT_RELOAD will have valid reg for those positions
@ -1471,7 +1462,7 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
offset += genTypeSize(type); offset += genTypeSize(type);
} }
// Updating variable liveness after instruction was emitted // Update variable liveness.
genUpdateLife(treeNode); genUpdateLife(treeNode);
varDsc->SetRegNum(REG_STK); varDsc->SetRegNum(REG_STK);
} }
@ -2333,132 +2324,6 @@ void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode)
genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN); genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN);
} }
//------------------------------------------------------------------------
// genRegCopy: Produce code for a GT_COPY node.
//
// Arguments:
// tree - the GT_COPY node
//
// Notes:
// This will copy the register(s) produced by this node's source, to
// the register(s) allocated to this GT_COPY node.
// It has some special handling for these cases:
// - when the source and target registers are in different register files
// (note that this is *not* a conversion).
// - when the source is a lclVar whose home location is being moved to a new
// register (rather than just being copied for temporary use).
//
void CodeGen::genRegCopy(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_COPY);
GenTree* op1 = treeNode->AsOp()->gtOp1;
regNumber sourceReg = genConsumeReg(op1);
if (op1->IsMultiRegNode())
{
noway_assert(!op1->IsCopyOrReload());
unsigned regCount = op1->GetMultiRegCount();
for (unsigned i = 0; i < regCount; i++)
{
regNumber srcReg = op1->GetRegByIndex(i);
regNumber tgtReg = treeNode->AsCopyOrReload()->GetRegNumByIdx(i);
var_types regType = op1->GetRegTypeByIndex(i);
inst_RV_RV(ins_Copy(regType), tgtReg, srcReg, regType);
}
}
else
{
var_types targetType = treeNode->TypeGet();
regNumber targetReg = treeNode->GetRegNum();
assert(targetReg != REG_NA);
assert(targetType != TYP_STRUCT);
// Check whether this node and the node from which we're copying the value have the same
// register type.
// This can happen if (currently iff) we have a SIMD vector type that fits in an integer
// register, in which case it is passed as an argument, or returned from a call,
// in an integer register and must be copied if it's in a floating point register.
bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1));
bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode));
if (srcFltReg != tgtFltReg)
{
#ifdef TARGET_ARM64
inst_RV_RV(INS_fmov, targetReg, sourceReg, targetType);
#else // !TARGET_ARM64
if (varTypeIsFloating(treeNode))
{
// GT_COPY from 'int' to 'float' currently can't happen. Maybe if ARM SIMD is implemented
// it will happen, according to the comment above?
NYI_ARM("genRegCopy from 'int' to 'float'");
}
else
{
assert(varTypeIsFloating(op1));
if (op1->TypeGet() == TYP_FLOAT)
{
inst_RV_RV(INS_vmov_f2i, targetReg, genConsumeReg(op1), targetType);
}
else
{
regNumber otherReg = (regNumber)treeNode->AsCopyOrReload()->gtOtherRegs[0];
assert(otherReg != REG_NA);
inst_RV_RV_RV(INS_vmov_d2i, targetReg, otherReg, genConsumeReg(op1), EA_8BYTE);
}
}
#endif // !TARGET_ARM64
}
else
{
inst_RV_RV(ins_Copy(targetType), targetReg, sourceReg, targetType);
}
}
if (op1->IsLocal())
{
// The lclVar will never be a def.
// If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will
// appropriately set the gcInfo for the copied value.
// If not, there are two cases we need to handle:
// - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable
// will remain live in its original register.
// genProduceReg() will appropriately set the gcInfo for the copied value,
// and genConsumeReg will reset it.
// - Otherwise, we need to update register info for the lclVar.
GenTreeLclVarCommon* lcl = op1->AsLclVarCommon();
assert((lcl->gtFlags & GTF_VAR_DEF) == 0);
if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0)
{
LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()];
// If we didn't just spill it (in genConsumeReg, above), then update the register info
if (varDsc->GetRegNum() != REG_STK)
{
// The old location is dying
genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1));
gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum()));
genUpdateVarReg(varDsc, treeNode);
#ifdef USING_VARIABLE_LIVE_RANGE
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum())
#endif // USING_VARIABLE_LIVE_RANGE
// The new location is going live
genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode));
}
}
}
genProduceReg(treeNode);
}
//------------------------------------------------------------------------ //------------------------------------------------------------------------
// genCallInstruction: Produce code for a GT_CALL node // genCallInstruction: Produce code for a GT_CALL node
// //
@ -3759,95 +3624,28 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea)
genProduceReg(lea); genProduceReg(lea);
} }
#ifdef FEATURE_SIMD
//------------------------------------------------------------------------ //------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct. // genSIMDSplitReturn: Generates code for returning a fixed-size SIMD type that lives
// in a single register, but is returned in multiple registers.
// //
// Arguments: // Arguments:
// treeNode - The tree node to evaluate whether is a struct return. // src - The source of the return
// retTypeDesc - The return type descriptor.
// //
// Return Value: void CodeGen::genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc)
// Returns true if the 'treeNode" is a GT_RETURN node of type struct.
// Otherwise returns false.
//
bool CodeGen::isStructReturn(GenTree* treeNode)
{ {
// This method could be called for 'treeNode' of GT_RET_FILT or GT_RETURN. assert(varTypeIsSIMD(src));
// For the GT_RET_FILT, the return is always assert(src->isUsedFromReg());
// a bool or a void, for the end of a finally block. regNumber srcReg = src->GetRegNum();
noway_assert(treeNode->OperGet() == GT_RETURN || treeNode->OperGet() == GT_RETFILT);
var_types returnType = treeNode->TypeGet();
#ifdef TARGET_ARM64
return varTypeIsStruct(returnType) && (compiler->info.compRetNativeType == TYP_STRUCT);
#else
return varTypeIsStruct(returnType);
#endif
}
//------------------------------------------------------------------------
// genStructReturn: Generates code for returning a struct.
//
// Arguments:
// treeNode - The GT_RETURN tree node.
//
// Return Value:
// None
//
// Assumption:
// op1 of GT_RETURN node is either GT_LCL_VAR or multi-reg GT_CALL
void CodeGen::genStructReturn(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_RETURN);
assert(isStructReturn(treeNode));
GenTree* op1 = treeNode->gtGetOp1();
if (op1->OperGet() == GT_LCL_VAR)
{
GenTreeLclVarCommon* lclVar = op1->AsLclVarCommon();
LclVarDsc* varDsc = &(compiler->lvaTable[lclVar->GetLclNum()]);
var_types lclType = genActualType(varDsc->TypeGet());
assert(varTypeIsStruct(lclType));
assert(varDsc->lvIsMultiRegRet);
ReturnTypeDesc retTypeDesc;
unsigned regCount;
retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle());
regCount = retTypeDesc.GetReturnRegCount();
assert(regCount >= 2);
assert(varTypeIsSIMD(lclType) || op1->isContained());
if (op1->isContained())
{
// Copy var on stack into ABI return registers
// TODO: It could be optimized by reducing two float loading to one double
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber reg = retTypeDesc.GetABIReturnReg(i);
GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), reg, lclVar->GetLclNum(), offset);
offset += genTypeSize(type);
}
}
else
{
// Handle SIMD genStructReturn case
NYI_ARM("SIMD genStructReturn");
#ifdef TARGET_ARM64
genConsumeRegs(op1);
regNumber src = op1->GetRegNum();
// Treat src register as a homogenous vector with element size equal to the reg size // Treat src register as a homogenous vector with element size equal to the reg size
// Insert pieces in order // Insert pieces in order
unsigned regCount = retTypeDesc->GetReturnRegCount();
for (unsigned i = 0; i < regCount; ++i) for (unsigned i = 0; i < regCount; ++i)
{ {
var_types type = retTypeDesc.GetReturnRegType(i); var_types type = retTypeDesc->GetReturnRegType(i);
regNumber reg = retTypeDesc.GetABIReturnReg(i); regNumber reg = retTypeDesc->GetABIReturnReg(i);
if (varTypeIsFloating(type)) if (varTypeIsFloating(type))
{ {
// If the register piece is to be passed in a floating point register // If the register piece is to be passed in a floating point register
@ -3857,7 +3655,7 @@ void CodeGen::genStructReturn(GenTree* treeNode)
// This effectively moves from `src[i]` to `reg[0]`, upper bits of reg remain unchanged // This effectively moves from `src[i]` to `reg[0]`, upper bits of reg remain unchanged
// For the case where src == reg, since we are only writing reg[0], as long as we iterate // For the case where src == reg, since we are only writing reg[0], as long as we iterate
// so that src[0] is consumed before writing reg[0], we do not need a temporary. // so that src[0] is consumed before writing reg[0], we do not need a temporary.
GetEmitter()->emitIns_R_R_I_I(INS_mov, emitTypeSize(type), reg, src, 0, i); GetEmitter()->emitIns_R_R_I_I(INS_mov, emitTypeSize(type), reg, srcReg, 0, i);
} }
else else
{ {
@ -3865,155 +3663,10 @@ void CodeGen::genStructReturn(GenTree* treeNode)
// Use a vector mov to general purpose register instruction // Use a vector mov to general purpose register instruction
// mov reg, src[i] // mov reg, src[i]
// This effectively moves from `src[i]` to `reg` // This effectively moves from `src[i]` to `reg`
GetEmitter()->emitIns_R_R_I(INS_mov, emitTypeSize(type), reg, src, i); GetEmitter()->emitIns_R_R_I(INS_mov, emitTypeSize(type), reg, srcReg, i);
}
}
#endif // TARGET_ARM64
}
}
else // op1 must be multi-reg GT_CALL
{
assert(op1->IsMultiRegCall() || op1->IsCopyOrReloadOfMultiRegCall());
genConsumeRegs(op1);
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc();
const unsigned regCount = pRetTypeDesc->GetReturnRegCount();
unsigned matchingCount = 0;
var_types regType[MAX_RET_REG_COUNT];
regNumber returnReg[MAX_RET_REG_COUNT];
regNumber allocatedReg[MAX_RET_REG_COUNT];
regMaskTP srcRegsMask = 0;
regMaskTP dstRegsMask = 0;
bool needToShuffleRegs = false; // Set to true if we have to move any registers
for (unsigned i = 0; i < regCount; ++i)
{
regType[i] = pRetTypeDesc->GetReturnRegType(i);
returnReg[i] = pRetTypeDesc->GetABIReturnReg(i);
regNumber reloadReg = REG_NA;
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
// that need to be copied or reloaded.
reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(i);
}
if (reloadReg != REG_NA)
{
allocatedReg[i] = reloadReg;
}
else
{
allocatedReg[i] = call->GetRegNumByIdx(i);
}
if (returnReg[i] == allocatedReg[i])
{
matchingCount++;
}
else // We need to move this value
{
// We want to move the value from allocatedReg[i] into returnReg[i]
// so record these two registers in the src and dst masks
//
srcRegsMask |= genRegMask(allocatedReg[i]);
dstRegsMask |= genRegMask(returnReg[i]);
needToShuffleRegs = true;
}
}
if (needToShuffleRegs)
{
assert(matchingCount < regCount);
unsigned remainingRegCount = regCount - matchingCount;
regMaskTP extraRegMask = treeNode->gtRsvdRegs;
while (remainingRegCount > 0)
{
// set 'available' to the 'dst' registers that are not currently holding 'src' registers
//
regMaskTP availableMask = dstRegsMask & ~srcRegsMask;
regMaskTP dstMask;
regNumber srcReg;
regNumber dstReg;
var_types curType = TYP_UNKNOWN;
regNumber freeUpReg = REG_NA;
if (availableMask == 0)
{
// Circular register dependencies
// So just free up the lowest register in dstRegsMask by moving it to the 'extra' register
assert(dstRegsMask == srcRegsMask); // this has to be true for us to reach here
assert(extraRegMask != 0); // we require an 'extra' register
assert((extraRegMask & ~dstRegsMask) != 0); // it can't be part of dstRegsMask
availableMask = extraRegMask & ~dstRegsMask;
regMaskTP srcMask = genFindLowestBit(srcRegsMask);
freeUpReg = genRegNumFromMask(srcMask);
}
dstMask = genFindLowestBit(availableMask);
dstReg = genRegNumFromMask(dstMask);
srcReg = REG_NA;
if (freeUpReg != REG_NA)
{
// We will free up the srcReg by moving it to dstReg which is an extra register
//
srcReg = freeUpReg;
// Find the 'srcReg' and set 'curType', change allocatedReg[] to dstReg
// and add the new register mask bit to srcRegsMask
//
for (unsigned i = 0; i < regCount; ++i)
{
if (allocatedReg[i] == srcReg)
{
curType = regType[i];
allocatedReg[i] = dstReg;
srcRegsMask |= genRegMask(dstReg);
} }
} }
} }
else // The normal case #endif // FEATURE_SIMD
{
// Find the 'srcReg' and set 'curType'
//
for (unsigned i = 0; i < regCount; ++i)
{
if (returnReg[i] == dstReg)
{
srcReg = allocatedReg[i];
curType = regType[i];
}
}
// After we perform this move we will have one less registers to setup
remainingRegCount--;
}
assert(curType != TYP_UNKNOWN);
inst_RV_RV(ins_Copy(curType), dstReg, srcReg, curType);
// Clear the appropriate bits in srcRegsMask and dstRegsMask
srcRegsMask &= ~genRegMask(srcReg);
dstRegsMask &= ~genRegMask(dstReg);
} // while (remainingRegCount > 0)
} // (needToShuffleRegs)
} // op1 must be multi-reg GT_CALL
}
#endif // TARGET_ARMARCH #endif // TARGET_ARMARCH

276
src/coreclr/src/jit/codegencommon.cpp
View file

@ -11627,6 +11627,282 @@ void CodeGen::genReturn(GenTree* treeNode)
#endif // defined(DEBUG) && defined(TARGET_XARCH) #endif // defined(DEBUG) && defined(TARGET_XARCH)
} }
//------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct.
//
// Arguments:
// treeNode - The tree node to evaluate whether is a struct return.
//
// Return Value:
// Returns true if the 'treeNode" is a GT_RETURN node of type struct.
// Otherwise returns false.
//
bool CodeGen::isStructReturn(GenTree* treeNode)
{
// This method could be called for 'treeNode' of GT_RET_FILT or GT_RETURN.
// For the GT_RET_FILT, the return is always a bool or a void, for the end of a finally block.
noway_assert(treeNode->OperGet() == GT_RETURN || treeNode->OperGet() == GT_RETFILT);
if (treeNode->OperGet() != GT_RETURN)
{
return false;
}
#if defined(TARGET_AMD64) && !defined(UNIX_AMD64_ABI)
assert(!varTypeIsStruct(treeNode));
return false;
#elif defined(TARGET_ARM64)
return varTypeIsStruct(treeNode) && (compiler->info.compRetNativeType == TYP_STRUCT);
#else
return varTypeIsStruct(treeNode);
#endif
}
//------------------------------------------------------------------------
// genStructReturn: Generates code for returning a struct.
//
// Arguments:
// treeNode - The GT_RETURN tree node.
//
// Return Value:
// None
//
// Assumption:
// op1 of GT_RETURN node is either GT_LCL_VAR or multi-reg GT_CALL
//
void CodeGen::genStructReturn(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_RETURN);
GenTree* op1 = treeNode->gtGetOp1();
genConsumeRegs(op1);
GenTree* actualOp1 = op1;
if (op1->IsCopyOrReload())
{
actualOp1 = op1->gtGetOp1();
}
ReturnTypeDesc retTypeDesc;
LclVarDsc* varDsc = nullptr;
if (actualOp1->OperIs(GT_LCL_VAR))
{
varDsc = compiler->lvaGetDesc(actualOp1->AsLclVar()->GetLclNum());
retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle());
}
else
{
assert(actualOp1->OperIs(GT_CALL));
retTypeDesc = *(actualOp1->AsCall()->GetReturnTypeDesc());
}
unsigned regCount = retTypeDesc.GetReturnRegCount();
assert(regCount <= MAX_RET_REG_COUNT);
#if FEATURE_MULTIREG_RET
if (actualOp1->OperIs(GT_LCL_VAR) && (varTypeIsEnregisterable(op1)))
{
// Right now the only enregisterable structs supported are SIMD vector types.
assert(varTypeIsSIMD(op1));
#ifdef FEATURE_SIMD
genSIMDSplitReturn(op1, &retTypeDesc);
#endif // FEATURE_SIMD
}
else if (actualOp1->OperIs(GT_LCL_VAR))
{
GenTreeLclVar* lclNode = actualOp1->AsLclVar();
LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum());
assert(varDsc->lvIsMultiRegRet);
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber toReg = retTypeDesc.GetABIReturnReg(i);
GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), toReg, lclNode->GetLclNum(), offset);
offset += genTypeSize(type);
}
}
else
{
assert(actualOp1->IsMultiRegCall());
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber toReg = retTypeDesc.GetABIReturnReg(i);
regNumber fromReg = op1->GetRegByIndex(i);
if (fromReg == REG_NA)
{
assert(op1->IsCopyOrReload());
fromReg = actualOp1->GetRegByIndex(i);
}
if (fromReg != toReg)
{
inst_RV_RV(ins_Copy(type), toReg, fromReg, type);
}
}
}
#else // !FEATURE_MULTIREG_RET
unreached();
#endif
}
//------------------------------------------------------------------------
// genRegCopy: Produce code for a GT_COPY node.
//
// Arguments:
// tree - the GT_COPY node
//
// Notes:
// This will copy the register(s) produced by this nodes source, to
// the register(s) allocated to this GT_COPY node.
// It has some special handling for these casess:
// - when the source and target registers are in different register files
// (note that this is *not* a conversion).
// - when the source is a lclVar whose home location is being moved to a new
// register (rather than just being copied for temporary use).
//
void CodeGen::genRegCopy(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_COPY);
GenTree* op1 = treeNode->AsOp()->gtOp1;
if (op1->IsMultiRegNode())
{
// Register allocation assumes that any reload and copy are done in operand order.
// That is, we can have:
// (reg0, reg1) = COPY(V0,V1) where V0 is in reg1 and V1 is in memory
// The register allocation model assumes:
// First, V0 is moved to reg0 (v1 can't be in reg0 because it is still live, which would be a conflict).
// Then, V1 is moved to reg1
// However, if we call genConsumeRegs on op1, it will do the reload of V1 before we do the copy of V0.
// So we need to handle that case first.
//
// There should never be any circular dependencies, and we will check that here.
GenTreeCopyOrReload* copyNode = treeNode->AsCopyOrReload();
unsigned regCount = copyNode->GetRegCount();
// GenTreeCopyOrReload only reports the number of registers that are valid.
assert(regCount <= 2);
// First set the source registers as busy if they haven't been spilled.
// (Note that this is just for verification that we don't have circular dependencies.)
regMaskTP busyRegs = RBM_NONE;
for (unsigned i = 0; i < regCount; ++i)
{
if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0)
{
busyRegs |= genRegMask(op1->GetRegByIndex(i));
}
}
// First do any copies - we'll do the reloads after all the copies are complete.
for (unsigned i = 0; i < regCount; ++i)
{
regNumber sourceReg = op1->GetRegByIndex(i);
regNumber targetReg = copyNode->GetRegNumByIdx(i);
regMaskTP targetRegMask = genRegMask(targetReg);
// GenTreeCopyOrReload only reports the number of registers that are valid.
if (targetReg != REG_NA)
{
// We shouldn't specify a no-op move.
assert(sourceReg != targetReg);
assert((busyRegs & targetRegMask) == 0);
// Clear sourceReg from the busyRegs, and add targetReg.
busyRegs &= ~genRegMask(sourceReg);
busyRegs |= genRegMask(targetReg);
var_types type;
if (op1->IsMultiRegLclVar())
{
type = op1->AsLclVar()->GetFieldTypeByIndex(compiler, i);
}
else
{
type = op1->GetRegTypeByIndex(i);
}
inst_RV_RV(ins_Copy(type), targetReg, sourceReg, type);
}
}
// Now we can consume op1, which will perform any necessary reloads.
genConsumeReg(op1);
}
else
{
var_types targetType = treeNode->TypeGet();
regNumber targetReg = treeNode->GetRegNum();
assert(targetReg != REG_NA);
assert(targetType != TYP_STRUCT);
// Check whether this node and the node from which we're copying the value have
// different register types. This can happen if (currently iff) we have a SIMD
// vector type that fits in an integer register, in which case it is passed as
// an argument, or returned from a call, in an integer register and must be
// copied if it's in an xmm register.
bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1));
bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode));
if (srcFltReg != tgtFltReg)
{
instruction ins;
regNumber fpReg;
regNumber intReg;
if (tgtFltReg)
{
ins = ins_CopyIntToFloat(op1->TypeGet(), treeNode->TypeGet());
fpReg = targetReg;
intReg = op1->GetRegNum();
}
else
{
ins = ins_CopyFloatToInt(op1->TypeGet(), treeNode->TypeGet());
intReg = targetReg;
fpReg = op1->GetRegNum();
}
inst_RV_RV(ins, fpReg, intReg, targetType);
}
else
{
inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType);
}
if (op1->IsLocal())
{
// The lclVar will never be a def.
// If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will
// appropriately set the gcInfo for the copied value.
// If not, there are two cases we need to handle:
// - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable
// will remain live in its original register.
// genProduceReg() will appropriately set the gcInfo for the copied value,
// and genConsumeReg will reset it.
// - Otherwise, we need to update register info for the lclVar.
GenTreeLclVarCommon* lcl = op1->AsLclVarCommon();
assert((lcl->gtFlags & GTF_VAR_DEF) == 0);
if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0)
{
LclVarDsc* varDsc = compiler->lvaGetDesc(lcl);
// If we didn't just spill it (in genConsumeReg, above), then update the register info
if (varDsc->GetRegNum() != REG_STK)
{
// The old location is dying
genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1));
gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum()));
genUpdateVarReg(varDsc, treeNode);
#ifdef USING_VARIABLE_LIVE_RANGE
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum());
#endif // USING_VARIABLE_LIVE_RANGE
// The new location is going live
genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode));
}
}
}
}
genProduceReg(treeNode);
}
#if defined(DEBUG) && defined(TARGET_XARCH) #if defined(DEBUG) && defined(TARGET_XARCH)
//------------------------------------------------------------------------ //------------------------------------------------------------------------

231
src/coreclr/src/jit/codegenlinear.cpp
View file

@ -938,6 +938,86 @@ GenTree* sameRegAsDst(GenTree* tree, GenTree*& other /*out*/)
} }
} }
//------------------------------------------------------------------------
// genUnspillLocal: Reload a register candidate local into a register.
//
// Arguments:
// varNum - The variable number of the local to be reloaded (unspilled).
// It may be a local field.
// type - The type of the local.
// lclNode - The node being unspilled. Note that for a multi-reg local,
// the gtLclNum will be that of the parent struct.
// regNum - The register that 'varNum' should be loaded to.
// reSpill - True if it will be immediately spilled after use.
// isLastUse - True if this is a last use of 'varNum'.
//
// Notes:
// The caller must have determined that this local needs to be unspilled.
void CodeGen::genUnspillLocal(
unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse)
{
LclVarDsc* varDsc = compiler->lvaGetDesc(varNum);
inst_set_SV_var(lclNode);
instruction ins = ins_Load(type, compiler->isSIMDTypeLocalAligned(varNum));
GetEmitter()->emitIns_R_S(ins, emitTypeSize(type), regNum, varNum, 0);
// TODO-Review: We would like to call:
// genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(tree));
// instead of the following code, but this ends up hitting this assert:
// assert((regSet.GetMaskVars() & regMask) == 0);
// due to issues with LSRA resolution moves.
// So, just force it for now. This probably indicates a condition that creates a GC hole!
//
// Extra note: I think we really want to call something like gcInfo.gcUpdateForRegVarMove,
// because the variable is not really going live or dead, but that method is somewhat poorly
// factored because it, in turn, updates rsMaskVars which is part of RegSet not GCInfo.
// TODO-Cleanup: This code exists in other CodeGen*.cpp files, and should be moved to CodeGenCommon.cpp.
// Don't update the variable's location if we are just re-spilling it again.
if (!reSpill)
{
varDsc->SetRegNum(regNum);
#ifdef USING_VARIABLE_LIVE_RANGE
// We want "VariableLiveRange" inclusive on the beginning and exclusive on the ending.
// For that we shouldn't report an update of the variable location if is becoming dead
// on the same native offset.
if (!isLastUse)
{
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, varNum);
}
#endif // USING_VARIABLE_LIVE_RANGE
if (!varDsc->lvLiveInOutOfHndlr)
{
#ifdef DEBUG
if (VarSetOps::IsMember(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex))
{
JITDUMP("\t\t\t\t\t\t\tRemoving V%02u from gcVarPtrSetCur\n", varNum);
}
#endif // DEBUG
VarSetOps::RemoveElemD(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex);
}
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tV%02u in reg ", varNum);
varDsc->PrintVarReg();
printf(" is becoming live ");
compiler->printTreeID(lclNode);
printf("\n");
}
#endif // DEBUG
regSet.AddMaskVars(genGetRegMask(varDsc));
}
gcInfo.gcMarkRegPtrVal(regNum, type);
}
//------------------------------------------------------------------------ //------------------------------------------------------------------------
// genUnspillRegIfNeeded: Reload the value into a register, if needed // genUnspillRegIfNeeded: Reload the value into a register, if needed
// //
@ -968,8 +1048,9 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree)
// Reset spilled flag, since we are going to load a local variable from its home location. // Reset spilled flag, since we are going to load a local variable from its home location.
unspillTree->gtFlags &= ~GTF_SPILLED; unspillTree->gtFlags &= ~GTF_SPILLED;
GenTreeLclVarCommon* lcl = unspillTree->AsLclVarCommon(); GenTreeLclVar* lcl = unspillTree->AsLclVar();
LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()]; LclVarDsc* varDsc = compiler->lvaGetDesc(lcl->GetLclNum());
var_types spillType = unspillTree->TypeGet();
// TODO-Cleanup: The following code could probably be further merged and cleaned up. // TODO-Cleanup: The following code could probably be further merged and cleaned up.
#ifdef TARGET_XARCH #ifdef TARGET_XARCH
@ -985,99 +1066,26 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree)
// In the normalizeOnLoad case ins_Load will return an appropriate sign- or zero- // In the normalizeOnLoad case ins_Load will return an appropriate sign- or zero-
// extending load. // extending load.
var_types treeType = unspillTree->TypeGet(); if (spillType != genActualType(varDsc->lvType) && !varTypeIsGC(spillType) && !varDsc->lvNormalizeOnLoad())
if (treeType != genActualType(varDsc->lvType) && !varTypeIsGC(treeType) && !varDsc->lvNormalizeOnLoad())
{ {
assert(!varTypeIsGC(varDsc)); assert(!varTypeIsGC(varDsc));
var_types spillType = genActualType(varDsc->lvType); spillType = genActualType(varDsc->lvType);
unspillTree->gtType = spillType;
inst_RV_TT(ins_Load(spillType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum())), dstReg,
unspillTree);
unspillTree->gtType = treeType;
}
else
{
inst_RV_TT(ins_Load(treeType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum())), dstReg, unspillTree);
} }
#elif defined(TARGET_ARM64) #elif defined(TARGET_ARM64)
var_types targetType = unspillTree->gtType; var_types targetType = unspillTree->gtType;
if (targetType != genActualType(varDsc->lvType) && !varTypeIsGC(targetType) && !varDsc->lvNormalizeOnLoad()) if (spillType != genActualType(varDsc->lvType) && !varTypeIsGC(spillType) && !varDsc->lvNormalizeOnLoad())
{ {
assert(!varTypeIsGC(varDsc)); assert(!varTypeIsGC(varDsc));
targetType = genActualType(varDsc->lvType); spillType = genActualType(varDsc->lvType);
} }
instruction ins = ins_Load(targetType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum()));
emitAttr attr = emitActualTypeSize(targetType);
emitter* emit = GetEmitter();
// Load local variable from its home location.
inst_RV_TT(ins, dstReg, unspillTree, 0, attr);
#elif defined(TARGET_ARM) #elif defined(TARGET_ARM)
var_types targetType = unspillTree->gtType; // No normalizing for ARM
instruction ins = ins_Load(targetType, compiler->isSIMDTypeLocalAligned(lcl->GetLclNum()));
emitAttr attr = emitTypeSize(targetType);
// Load local variable from its home location.
inst_RV_TT(ins, dstReg, unspillTree, 0, attr);
#else #else
NYI("Unspilling not implemented for this target architecture."); NYI("Unspilling not implemented for this target architecture.");
#endif #endif
bool reSpill = ((unspillTree->gtFlags & GTF_SPILL) != 0);
// TODO-Review: We would like to call: bool isLastUse = lcl->IsLastUse(0);
// genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(tree)); genUnspillLocal(lcl->GetLclNum(), spillType, lcl, dstReg, reSpill, isLastUse);
// instead of the following code, but this ends up hitting this assert:
// assert((regSet.GetMaskVars() & regMask) == 0);
// due to issues with LSRA resolution moves.
// So, just force it for now. This probably indicates a condition that creates a GC hole!
//
// Extra note: I think we really want to call something like gcInfo.gcUpdateForRegVarMove,
// because the variable is not really going live or dead, but that method is somewhat poorly
// factored because it, in turn, updates rsMaskVars which is part of RegSet not GCInfo.
// TODO-Cleanup: This code exists in other CodeGen*.cpp files, and should be moved to CodeGenCommon.cpp.
// Don't update the variable's location if we are just re-spilling it again.
if ((unspillTree->gtFlags & GTF_SPILL) == 0)
{
genUpdateVarReg(varDsc, tree);
#ifdef USING_VARIABLE_LIVE_RANGE
// We want "VariableLiveRange" inclusive on the beginbing and exclusive on the ending.
// For that we shouldn't report an update of the variable location if is becoming dead
// on the same native offset.
if ((unspillTree->gtFlags & GTF_VAR_DEATH) == 0)
{
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum());
}
#endif // USING_VARIABLE_LIVE_RANGE
if (!varDsc->lvLiveInOutOfHndlr)
{
#ifdef DEBUG
if (VarSetOps::IsMember(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex))
{
JITDUMP("\t\t\t\t\t\t\tRemoving V%02u from gcVarPtrSetCur\n", lcl->GetLclNum());
}
#endif // DEBUG
VarSetOps::RemoveElemD(compiler, gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex);
}
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tV%02u in reg ", lcl->GetLclNum());
varDsc->PrintVarReg();
printf(" is becoming live ");
compiler->printTreeID(unspillTree);
printf("\n");
}
#endif // DEBUG
regSet.AddMaskVars(genGetRegMask(varDsc));
}
gcInfo.gcMarkRegPtrVal(dstReg, unspillTree->TypeGet());
} }
else if (unspillTree->IsMultiRegCall()) else if (unspillTree->IsMultiRegCall())
{ {
@ -1849,6 +1857,41 @@ void CodeGen::genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber
genSetBlockSize(blkNode, sizeReg); genSetBlockSize(blkNode, sizeReg);
} }
//-------------------------------------------------------------------------
// genSpillLocal: Generate the actual spill of a local var.
//
// Arguments:
// varNum - The variable number of the local to be spilled.
// It may be a local field.
// type - The type of the local.
// lclNode - The node being spilled. Note that for a multi-reg local,
// the gtLclNum will be that of the parent struct.
// regNum - The register that 'varNum' is currently in.
//
// Return Value:
// None.
//
void CodeGen::genSpillLocal(unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum)
{
LclVarDsc* varDsc = compiler->lvaGetDesc(varNum);
assert(!varDsc->lvNormalizeOnStore() || (type == genActualType(varDsc->TypeGet())));
// We have a register candidate local that is marked with GTF_SPILL.
// This flag generally means that we need to spill this local.
// The exception is the case of a use of an EH var use that is being "spilled"
// to the stack, indicated by GTF_SPILL (note that all EH lclVar defs are always
// spilled, i.e. write-thru).
// An EH var use is always valid on the stack (so we don't need to actually spill it),
// but the GTF_SPILL flag records the fact that the register value is going dead.
if (((lclNode->gtFlags & GTF_VAR_DEF) != 0) || !varDsc->lvLiveInOutOfHndlr)
{
// Store local variable to its home location.
// Ensure that lclVar stores are typed correctly.
GetEmitter()->emitIns_S_R(ins_Store(type, compiler->isSIMDTypeLocalAligned(varNum)), emitTypeSize(type), regNum,
varNum, 0);
}
}
//------------------------------------------------------------------------- //-------------------------------------------------------------------------
// genProduceReg: do liveness update for register produced by the current // genProduceReg: do liveness update for register produced by the current
// node in codegen after code has been emitted for it. // node in codegen after code has been emitted for it.
@ -1878,23 +1921,7 @@ void CodeGen::genProduceReg(GenTree* tree)
if (genIsRegCandidateLocal(tree)) if (genIsRegCandidateLocal(tree))
{ {
unsigned varNum = tree->AsLclVarCommon()->GetLclNum(); unsigned varNum = tree->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = compiler->lvaGetDesc(varNum); genSpillLocal(varNum, tree->TypeGet(), tree->AsLclVar(), tree->GetRegNum());
assert(!varDsc->lvNormalizeOnStore() || (tree->TypeGet() == genActualType(varDsc->TypeGet())));
// If we reach here, we have a register candidate local that is marked with GTF_SPILL.
// This flag generally means that we need to spill this local.
// The exception is the case of a use of an EH var use that is being "spilled"
// to the stack, indicated by GTF_SPILL (note that all EH lclVar defs are always
// spilled, i.e. write-thru).
// An EH var use is always valid on the stack (so we don't need to actually spill it),
// but the GTF_SPILL flag records the fact that the register value is going dead.
if (((tree->gtFlags & GTF_VAR_DEF) != 0) || !varDsc->lvLiveInOutOfHndlr)
{
// Store local variable to its home location.
// Ensure that lclVar stores are typed correctly.
inst_TT_RV(ins_Store(tree->gtType, compiler->isSIMDTypeLocalAligned(varNum)),
emitTypeSize(tree->TypeGet()), tree, tree->GetRegNum());
}
} }
else else
{ {
@ -1983,8 +2010,8 @@ void CodeGen::genProduceReg(GenTree* tree)
// the register as live, with a GC pointer, if the variable is dead. // the register as live, with a GC pointer, if the variable is dead.
if (!genIsRegCandidateLocal(tree) || ((tree->gtFlags & GTF_VAR_DEATH) == 0)) if (!genIsRegCandidateLocal(tree) || ((tree->gtFlags & GTF_VAR_DEATH) == 0))
{ {
// Multi-reg call node will produce more than one register result. // Multi-reg nodes will produce more than one register result.
// Mark all the regs produced by call node. // Mark all the regs produced by the node.
if (tree->IsMultiRegCall()) if (tree->IsMultiRegCall())
{ {
const GenTreeCall* call = tree->AsCall(); const GenTreeCall* call = tree->AsCall();

431
src/coreclr/src/jit/codegenxarch.cpp
View file

@ -1124,75 +1124,25 @@ void CodeGen::genCodeForMul(GenTreeOp* treeNode)
genProduceReg(treeNode); genProduceReg(treeNode);
} }
#ifdef FEATURE_SIMD
//------------------------------------------------------------------------ //------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct. // genSIMDSplitReturn: Generates code for returning a fixed-size SIMD type that lives
// in a single register, but is returned in multiple registers.
// //
// Arguments: // Arguments:
// treeNode - The tree node to evaluate whether is a struct return. // src - The source of the return
// retTypeDesc - The return type descriptor.
// //
// Return Value: void CodeGen::genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc)
// For AMD64 *nix: returns true if the 'treeNode" is a GT_RETURN node, of type struct.
// Otherwise returns false.
// For other platforms always returns false.
//
bool CodeGen::isStructReturn(GenTree* treeNode)
{ {
// This method could be called for 'treeNode' of GT_RET_FILT or GT_RETURN. assert(varTypeIsSIMD(src));
// For the GT_RET_FILT, the return is always assert(src->isUsedFromReg());
// a bool or a void, for the end of a finally block.
noway_assert(treeNode->OperGet() == GT_RETURN || treeNode->OperGet() == GT_RETFILT);
if (treeNode->OperGet() != GT_RETURN)
{
return false;
}
#ifdef UNIX_AMD64_ABI
return varTypeIsStruct(treeNode);
#else // !UNIX_AMD64_ABI
assert(!varTypeIsStruct(treeNode));
return false;
#endif // UNIX_AMD64_ABI
}
//------------------------------------------------------------------------
// genStructReturn: Generates code for returning a struct.
//
// Arguments:
// treeNode - The GT_RETURN tree node.
//
// Return Value:
// None
//
// Assumption:
// op1 of GT_RETURN node is either GT_LCL_VAR or multi-reg GT_CALL
void CodeGen::genStructReturn(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_RETURN);
GenTree* op1 = treeNode->gtGetOp1();
#ifdef UNIX_AMD64_ABI
if (op1->OperGet() == GT_LCL_VAR)
{
GenTreeLclVarCommon* lclVar = op1->AsLclVarCommon();
LclVarDsc* varDsc = &(compiler->lvaTable[lclVar->GetLclNum()]);
assert(varDsc->lvIsMultiRegRet);
ReturnTypeDesc retTypeDesc;
retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle());
const unsigned regCount = retTypeDesc.GetReturnRegCount();
assert(regCount == MAX_RET_REG_COUNT);
if (varTypeIsEnregisterable(op1))
{
// Right now the only enregisterable structs supported are SIMD vector types.
assert(varTypeIsSIMD(op1));
assert(op1->isUsedFromReg());
// This is a case of operand is in a single reg and needs to be // This is a case of operand is in a single reg and needs to be
// returned in multiple ABI return registers. // returned in multiple ABI return registers.
regNumber opReg = genConsumeReg(op1); regNumber opReg = src->GetRegNum();
regNumber reg0 = retTypeDesc.GetABIReturnReg(0); regNumber reg0 = retTypeDesc->GetABIReturnReg(0);
regNumber reg1 = retTypeDesc.GetABIReturnReg(1); regNumber reg1 = retTypeDesc->GetABIReturnReg(1);
if (opReg != reg0 && opReg != reg1) if (opReg != reg0 && opReg != reg1)
{ {
@ -1222,116 +1172,7 @@ void CodeGen::genStructReturn(GenTree* treeNode)
} }
inst_RV_RV_IV(INS_shufpd, EA_16BYTE, reg1, reg1, 0x01); inst_RV_RV_IV(INS_shufpd, EA_16BYTE, reg1, reg1, 0x01);
} }
else #endif // FEATURE_SIMD
{
assert(op1->isUsedFromMemory());
// Copy var on stack into ABI return registers
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc.GetReturnRegType(i);
regNumber reg = retTypeDesc.GetABIReturnReg(i);
GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), reg, lclVar->GetLclNum(), offset);
offset += genTypeSize(type);
}
}
}
else
{
assert(op1->IsMultiRegCall() || op1->IsCopyOrReloadOfMultiRegCall());
genConsumeRegs(op1);
const GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
const GenTreeCall* call = actualOp1->AsCall();
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
const unsigned regCount = retTypeDesc->GetReturnRegCount();
assert(regCount == MAX_RET_REG_COUNT);
// Handle circular dependency between call allocated regs and ABI return regs.
//
// It is possible under LSRA stress that originally allocated regs of call node,
// say rax and rdx, are spilled and reloaded to rdx and rax respectively. But
// GT_RETURN needs to move values as follows: rdx->rax, rax->rdx. Similar kind
// kind of circular dependency could arise between xmm0 and xmm1 return regs.
// Codegen is expected to handle such circular dependency.
//
var_types regType0 = retTypeDesc->GetReturnRegType(0);
regNumber returnReg0 = retTypeDesc->GetABIReturnReg(0);
regNumber allocatedReg0 = call->GetRegNumByIdx(0);
var_types regType1 = retTypeDesc->GetReturnRegType(1);
regNumber returnReg1 = retTypeDesc->GetABIReturnReg(1);
regNumber allocatedReg1 = call->GetRegNumByIdx(1);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
// that need to be copied or reloaded.
regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(0);
if (reloadReg != REG_NA)
{
allocatedReg0 = reloadReg;
}
reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(1);
if (reloadReg != REG_NA)
{
allocatedReg1 = reloadReg;
}
}
if (allocatedReg0 == returnReg1 && allocatedReg1 == returnReg0)
{
// Circular dependency - swap allocatedReg0 and allocatedReg1
if (varTypeIsFloating(regType0))
{
assert(varTypeIsFloating(regType1));
// The fastest way to swap two XMM regs is using PXOR
inst_RV_RV(INS_pxor, allocatedReg0, allocatedReg1, TYP_DOUBLE);
inst_RV_RV(INS_pxor, allocatedReg1, allocatedReg0, TYP_DOUBLE);
inst_RV_RV(INS_pxor, allocatedReg0, allocatedReg1, TYP_DOUBLE);
}
else
{
assert(varTypeIsIntegral(regType0));
assert(varTypeIsIntegral(regType1));
inst_RV_RV(INS_xchg, allocatedReg1, allocatedReg0, TYP_I_IMPL);
}
}
else if (allocatedReg1 == returnReg0)
{
// Change the order of moves to correctly handle dependency.
if (allocatedReg1 != returnReg1)
{
inst_RV_RV(ins_Copy(regType1), returnReg1, allocatedReg1, regType1);
}
if (allocatedReg0 != returnReg0)
{
inst_RV_RV(ins_Copy(regType0), returnReg0, allocatedReg0, regType0);
}
}
else
{
// No circular dependency case.
if (allocatedReg0 != returnReg0)
{
inst_RV_RV(ins_Copy(regType0), returnReg0, allocatedReg0, regType0);
}
if (allocatedReg1 != returnReg1)
{
inst_RV_RV(ins_Copy(regType1), returnReg1, allocatedReg1, regType1);
}
}
}
#else
unreached();
#endif
}
#if defined(TARGET_X86) #if defined(TARGET_X86)
@ -2009,7 +1850,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
} }
//---------------------------------------------------------------------------------- //----------------------------------------------------------------------------------
// genMultiRegCallStoreToLocal: store multi-reg return value of a call node to a local // genMultiRegStoreToLocal: store multi-reg return value of a call node to a local
// //
// Arguments: // Arguments:
// treeNode - Gentree of GT_STORE_LCL_VAR // treeNode - Gentree of GT_STORE_LCL_VAR
@ -2017,45 +1858,52 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
// Return Value: // Return Value:
// None // None
// //
// Assumption: // Assumptions:
// The child of store is a multi-reg call node. // The child of store is a multi-reg node.
// genProduceReg() on treeNode is made by caller of this routine.
// //
void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode) void CodeGen::genMultiRegStoreToLocal(GenTree* treeNode)
{ {
assert(treeNode->OperGet() == GT_STORE_LCL_VAR); assert(treeNode->OperGet() == GT_STORE_LCL_VAR);
assert(varTypeIsStruct(treeNode) || varTypeIsMultiReg(treeNode));
#ifdef UNIX_AMD64_ABI
// Structs of size >=9 and <=16 are returned in two return registers on x64 Unix.
assert(varTypeIsStruct(treeNode));
// Assumption: current x64 Unix implementation requires that a multi-reg struct
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being struct promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = &(compiler->lvaTable[lclNum]);
noway_assert(varDsc->lvIsMultiRegRet);
GenTree* op1 = treeNode->gtGetOp1(); GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy(); GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall(); assert(op1->IsMultiRegNode());
assert(call->HasMultiRegRetVal()); unsigned regCount = op1->GetMultiRegCount();
// Assumption: The current implementation requires that a multi-reg
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being struct promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum);
if (op1->OperIs(GT_CALL))
{
assert(regCount == MAX_RET_REG_COUNT);
noway_assert(varDsc->lvIsMultiRegRet);
}
genConsumeRegs(op1); genConsumeRegs(op1);
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); #ifdef UNIX_AMD64_ABI
assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT); // Structs of size >=9 and <=16 are returned in two return registers on x64 Unix.
unsigned regCount = retTypeDesc->GetReturnRegCount();
if (treeNode->GetRegNum() != REG_NA) // Handle the case of a SIMD type returned in 2 registers.
if (varTypeIsSIMD(treeNode) && (treeNode->GetRegNum() != REG_NA))
{ {
// Right now the only enregistrable structs supported are SIMD types. // Right now the only enregistrable structs supported are SIMD types.
assert(varTypeIsSIMD(treeNode)); // They are only returned in 1 or 2 registers - the 1 register case is
// handled as a regular STORE_LCL_VAR.
// This case is always a call (AsCall() will assert if it is not).
GenTreeCall* call = actualOp1->AsCall();
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT);
assert(regCount == 2);
assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(0))); assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(0)));
assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(1))); assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(1)));
// This is a case of two 8-bytes that comprise the operand is in // This is a case where the two 8-bytes that comprise the operand are in
// two different xmm registers and needs to assembled into a single // two different xmm registers and need to be assembled into a single
// xmm register. // xmm register.
regNumber targetReg = treeNode->GetRegNum(); regNumber targetReg = treeNode->GetRegNum();
regNumber reg0 = call->GetRegNumByIdx(0); regNumber reg0 = call->GetRegNumByIdx(0);
@ -2111,13 +1959,17 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
} }
} }
else else
#endif // UNIX_AMD64_ABI
{ {
// Stack store // This may be:
// - a call returning multiple registers
// - a HW intrinsic producing two registers to be stored into a TYP_STRUCT
//
int offset = 0; int offset = 0;
for (unsigned i = 0; i < regCount; ++i) for (unsigned i = 0; i < regCount; ++i)
{ {
var_types type = retTypeDesc->GetReturnRegType(i); var_types type = op1->GetRegTypeByIndex(i);
regNumber reg = call->GetRegNumByIdx(i); regNumber reg = op1->GetRegByIndex(i);
if (op1->IsCopyOrReload()) if (op1->IsCopyOrReload())
{ {
// GT_COPY/GT_RELOAD will have valid reg for those positions // GT_COPY/GT_RELOAD will have valid reg for those positions
@ -2133,57 +1985,10 @@ void CodeGen::genMultiRegCallStoreToLocal(GenTree* treeNode)
GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset); GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset);
offset += genTypeSize(type); offset += genTypeSize(type);
} }
// Update variable liveness.
genUpdateLife(treeNode);
varDsc->SetRegNum(REG_STK); varDsc->SetRegNum(REG_STK);
} }
#elif defined(TARGET_X86)
// Longs are returned in two return registers on x86.
assert(varTypeIsLong(treeNode));
// Assumption: current x86 implementation requires that a multi-reg long
// var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from
// being promoted.
unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum();
LclVarDsc* varDsc = &(compiler->lvaTable[lclNum]);
noway_assert(varDsc->lvIsMultiRegRet);
GenTree* op1 = treeNode->gtGetOp1();
GenTree* actualOp1 = op1->gtSkipReloadOrCopy();
GenTreeCall* call = actualOp1->AsCall();
assert(call->HasMultiRegRetVal());
genConsumeRegs(op1);
const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc();
unsigned regCount = retTypeDesc->GetReturnRegCount();
assert(regCount == MAX_RET_REG_COUNT);
// Stack store
int offset = 0;
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = retTypeDesc->GetReturnRegType(i);
regNumber reg = call->GetRegNumByIdx(i);
if (op1->IsCopyOrReload())
{
// GT_COPY/GT_RELOAD will have valid reg for those positions
// that need to be copied or reloaded.
regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(i);
if (reloadReg != REG_NA)
{
reg = reloadReg;
}
}
assert(reg != REG_NA);
GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset);
offset += genTypeSize(type);
}
varDsc->SetRegNum(REG_STK);
#else // !UNIX_AMD64_ABI && !TARGET_X86
assert(!"Unreached");
#endif // !UNIX_AMD64_ABI && !TARGET_X86
} }
//------------------------------------------------------------------------ //------------------------------------------------------------------------
@ -4636,9 +4441,9 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree)
// var = call, where call returns a multi-reg return value // var = call, where call returns a multi-reg return value
// case is handled separately. // case is handled separately.
if (op1->gtSkipReloadOrCopy()->IsMultiRegCall()) if (op1->gtSkipReloadOrCopy()->IsMultiRegNode())
{ {
genMultiRegCallStoreToLocal(tree); genMultiRegStoreToLocal(tree);
} }
else else
{ {
@ -4874,130 +4679,6 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree)
genProduceReg(tree); genProduceReg(tree);
} }
//------------------------------------------------------------------------
// genRegCopy: Produce code for a GT_COPY node.
//
// Arguments:
// tree - the GT_COPY node
//
// Notes:
// This will copy the register(s) produced by this nodes source, to
// the register(s) allocated to this GT_COPY node.
// It has some special handling for these casess:
// - when the source and target registers are in different register files
// (note that this is *not* a conversion).
// - when the source is a lclVar whose home location is being moved to a new
// register (rather than just being copied for temporary use).
//
void CodeGen::genRegCopy(GenTree* treeNode)
{
assert(treeNode->OperGet() == GT_COPY);
GenTree* op1 = treeNode->AsOp()->gtOp1;
if (op1->IsMultiRegNode())
{
genConsumeReg(op1);
GenTreeCopyOrReload* copyTree = treeNode->AsCopyOrReload();
unsigned regCount = treeNode->GetMultiRegCount();
for (unsigned i = 0; i < regCount; ++i)
{
var_types type = op1->GetRegTypeByIndex(i);
regNumber fromReg = op1->GetRegByIndex(i);
regNumber toReg = copyTree->GetRegNumByIdx(i);
// A Multi-reg GT_COPY node will have a valid reg only for those positions for which a corresponding
// result reg of the multi-reg node needs to be copied.
if (toReg != REG_NA)
{
assert(toReg != fromReg);
inst_RV_RV(ins_Copy(type), toReg, fromReg, type);
}
}
}
else
{
var_types targetType = treeNode->TypeGet();
regNumber targetReg = treeNode->GetRegNum();
assert(targetReg != REG_NA);
// Check whether this node and the node from which we're copying the value have
// different register types. This can happen if (currently iff) we have a SIMD
// vector type that fits in an integer register, in which case it is passed as
// an argument, or returned from a call, in an integer register and must be
// copied if it's in an xmm register.
bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1));
bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode));
if (srcFltReg != tgtFltReg)
{
instruction ins;
regNumber fpReg;
regNumber intReg;
if (tgtFltReg)
{
ins = ins_CopyIntToFloat(op1->TypeGet(), treeNode->TypeGet());
fpReg = targetReg;
intReg = op1->GetRegNum();
}
else
{
ins = ins_CopyFloatToInt(op1->TypeGet(), treeNode->TypeGet());
intReg = targetReg;
fpReg = op1->GetRegNum();
}
inst_RV_RV(ins, fpReg, intReg, targetType);
}
else
{
inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType);
}
if (op1->IsLocal())
{
// The lclVar will never be a def.
// If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will
// appropriately set the gcInfo for the copied value.
// If not, there are two cases we need to handle:
// - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable
// will remain live in its original register.
// genProduceReg() will appropriately set the gcInfo for the copied value,
// and genConsumeReg will reset it.
// - Otherwise, we need to update register info for the lclVar.
GenTreeLclVarCommon* lcl = op1->AsLclVarCommon();
assert((lcl->gtFlags & GTF_VAR_DEF) == 0);
if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0)
{
LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()];
// If we didn't just spill it (in genConsumeReg, above), then update the register info
if (varDsc->GetRegNum() != REG_STK)
{
// The old location is dying
genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1));
gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum()));
genUpdateVarReg(varDsc, treeNode);
#ifdef USING_VARIABLE_LIVE_RANGE
// Report the home change for this variable
varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum());
#endif // USING_VARIABLE_LIVE_RANGE
// The new location is going live
genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode));
}
}
}
}
genProduceReg(treeNode);
}
//------------------------------------------------------------------------ //------------------------------------------------------------------------
// genCodeForStoreInd: Produce code for a GT_STOREIND node. // genCodeForStoreInd: Produce code for a GT_STOREIND node.
// //

70
src/coreclr/src/jit/gentree.h
View file

@ -775,14 +775,14 @@ public:
// Note that a node marked GTF_VAR_MULTIREG can only be a pure definition of all the fields, or a pure use of all the fields, // Note that a node marked GTF_VAR_MULTIREG can only be a pure definition of all the fields, or a pure use of all the fields,
// so we don't need the equivalent of GTF_VAR_USEASG. // so we don't need the equivalent of GTF_VAR_USEASG.
#define GTF_VAR_MULTIREG_DEATH0 0x20000000 // GT_LCL_VAR -- The last-use bit for a lclVar (the first register if it is multireg). #define GTF_VAR_MULTIREG_DEATH0 0x04000000 // GT_LCL_VAR -- The last-use bit for a lclVar (the first register if it is multireg).
#define GTF_VAR_DEATH GTF_VAR_MULTIREG_DEATH0 #define GTF_VAR_DEATH GTF_VAR_MULTIREG_DEATH0
#define GTF_VAR_MULTIREG_DEATH1 0x10000000 // GT_LCL_VAR -- The last-use bit for the second register of a multireg lclVar. #define GTF_VAR_MULTIREG_DEATH1 0x08000000 // GT_LCL_VAR -- The last-use bit for the second register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH2 0x08000000 // GT_LCL_VAR -- The last-use bit for the third register of a multireg lclVar. #define GTF_VAR_MULTIREG_DEATH2 0x10000000 // GT_LCL_VAR -- The last-use bit for the third register of a multireg lclVar.
#define GTF_VAR_MULTIREG_DEATH3 0x04000000 // GT_LCL_VAR -- The last-use bit for the fourth register of a multireg lclVar. #define GTF_VAR_MULTIREG_DEATH3 0x20000000 // GT_LCL_VAR -- The last-use bit for the fourth register of a multireg lclVar.
#define GTF_VAR_DEATH_MASK (GTF_VAR_MULTIREG_DEATH0|GTF_VAR_MULTIREG_DEATH1 | GTF_VAR_MULTIREG_DEATH2 | GTF_VAR_MULTIREG_DEATH3) #define GTF_VAR_DEATH_MASK (GTF_VAR_MULTIREG_DEATH0|GTF_VAR_MULTIREG_DEATH1 | GTF_VAR_MULTIREG_DEATH2 | GTF_VAR_MULTIREG_DEATH3)
// This is the amount we have to shift, plus the regIndex, to get the last use bit we want. // This is the amount we have to shift, plus the regIndex, to get the last use bit we want.
#define MULTIREG_LAST_USE_SHIFT 17 #define MULTIREG_LAST_USE_SHIFT 26
#define GTF_VAR_MULTIREG 0x02000000 // This is a struct or (on 32-bit platforms) long variable that is used or defined #define GTF_VAR_MULTIREG 0x02000000 // This is a struct or (on 32-bit platforms) long variable that is used or defined
// to/from a multireg source or destination (e.g. a call arg or return, or an op // to/from a multireg source or destination (e.g. a call arg or return, or an op
// that returns its result in multiple registers such as a long multiply). // that returns its result in multiple registers such as a long multiply).
@ -1733,6 +1733,9 @@ public:
// Returns the type of the regIndex'th register defined by a multi-reg node. // Returns the type of the regIndex'th register defined by a multi-reg node.
var_types GetRegTypeByIndex(int regIndex); var_types GetRegTypeByIndex(int regIndex);
// Returns the GTF flag equivalent for the regIndex'th register of a multi-reg node.
unsigned int GetRegSpillFlagByIdx(int regIndex) const;
// Returns true if it is a GT_COPY or GT_RELOAD node // Returns true if it is a GT_COPY or GT_RELOAD node
inline bool IsCopyOrReload() const; inline bool IsCopyOrReload() const;
@ -3233,6 +3236,7 @@ private:
unsigned int GetLastUseBit(int regIndex) unsigned int GetLastUseBit(int regIndex)
{ {
assert(regIndex < 4); assert(regIndex < 4);
static_assert_no_msg((1 << MULTIREG_LAST_USE_SHIFT) == GTF_VAR_MULTIREG_DEATH0);
return (1 << (MULTIREG_LAST_USE_SHIFT + regIndex)); return (1 << (MULTIREG_LAST_USE_SHIFT + regIndex));
} }
@ -3251,6 +3255,7 @@ public:
void SetMultiReg() void SetMultiReg()
{ {
gtFlags |= GTF_VAR_MULTIREG; gtFlags |= GTF_VAR_MULTIREG;
ClearOtherRegFlags();
} }
regNumber GetRegNumByIdx(int regIndex) regNumber GetRegNumByIdx(int regIndex)
@ -7303,6 +7308,61 @@ inline var_types GenTree::GetRegTypeByIndex(int regIndex)
return TYP_UNDEF; return TYP_UNDEF;
} }
//-----------------------------------------------------------------------------------
// GetRegSpillFlagByIdx: Get a specific register's spill flags, based on regIndex,
// for this multi-reg node.
//
// Arguments:
// regIndex - which register's spill flags to return
//
// Return Value:
// The spill flags (GTF_SPILL GTF_SPILLED) for this register.
//
// Notes:
// This must be a multireg node and 'regIndex' must be a valid index for this node.
// This method returns the GTF "equivalent" flags based on the packed flags on the multireg node.
//
inline unsigned int GenTree::GetRegSpillFlagByIdx(int regIndex) const
{
#if FEATURE_MULTIREG_RET
if (IsMultiRegCall())
{
return AsCall()->AsCall()->GetRegSpillFlagByIdx(regIndex);
}
#if FEATURE_ARG_SPLIT
if (OperIsPutArgSplit())
{
return AsPutArgSplit()->GetRegSpillFlagByIdx(regIndex);
}
#endif
#if !defined(TARGET_64BIT)
if (OperIsMultiRegOp())
{
return AsMultiRegOp()->GetRegSpillFlagByIdx(regIndex);
}
#endif
#endif // FEATURE_MULTIREG_RET
#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS)
if (OperIs(GT_HWINTRINSIC))
{
// At this time, the only multi-reg HW intrinsics all return the type of their
// arguments. If this changes, we will need a way to record or determine this.
assert(TypeGet() == TYP_STRUCT);
return gtGetOp1()->TypeGet();
}
#endif
if (OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR))
{
return AsLclVar()->GetRegSpillFlagByIdx(regIndex);
}
assert(!"Invalid node type for GetRegSpillFlagByIdx");
return TYP_UNDEF;
}
//------------------------------------------------------------------------- //-------------------------------------------------------------------------
// IsCopyOrReload: whether this is a GT_COPY or GT_RELOAD node. // IsCopyOrReload: whether this is a GT_COPY or GT_RELOAD node.
// //

7
src/coreclr/src/jit/lclvars.cpp
View file

@ -1971,11 +1971,13 @@ bool Compiler::StructPromotionHelper::ShouldPromoteStructVar(unsigned lclNum)
// //
void Compiler::StructPromotionHelper::SortStructFields() void Compiler::StructPromotionHelper::SortStructFields()
{ {
assert(!structPromotionInfo.fieldsSorted); if (!structPromotionInfo.fieldsSorted)
{
qsort(structPromotionInfo.fields, structPromotionInfo.fieldCnt, sizeof(*structPromotionInfo.fields), qsort(structPromotionInfo.fields, structPromotionInfo.fieldCnt, sizeof(*structPromotionInfo.fields),
lvaFieldOffsetCmp); lvaFieldOffsetCmp);
structPromotionInfo.fieldsSorted = true; structPromotionInfo.fieldsSorted = true;
} }
}
//-------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------
// GetFieldInfo - get struct field information. // GetFieldInfo - get struct field information.
@ -2158,10 +2160,7 @@ void Compiler::StructPromotionHelper::PromoteStructVar(unsigned lclNum)
} }
#endif #endif
if (!structPromotionInfo.fieldsSorted)
{
SortStructFields(); SortStructFields();
}
for (unsigned index = 0; index < structPromotionInfo.fieldCnt; ++index) for (unsigned index = 0; index < structPromotionInfo.fieldCnt; ++index)
{ {

51
src/coreclr/src/jit/lsra.cpp
View file

@ -1791,6 +1791,8 @@ void LinearScan::identifyCandidates()
VarSetOps::AddElemD(compiler, fpMaybeCandidateVars, varDsc->lvVarIndex); VarSetOps::AddElemD(compiler, fpMaybeCandidateVars, varDsc->lvVarIndex);
} }
} }
JITDUMP(" ");
DBEXEC(VERBOSE, newInt->dump());
} }
else else
{ {
@ -6282,6 +6284,26 @@ void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval, Regi
#endif #endif
} }
//-----------------------------------------------------------------------------
// writeLocalReg: Write the register assignment for a GT_LCL_VAR node.
//
// Arguments:
// lclNode - The GT_LCL_VAR node
// varNum - The variable number for the register
// reg - The assigned register
//
// Return Value:
// None
//
void LinearScan::writeLocalReg(GenTreeLclVar* lclNode, unsigned varNum, regNumber reg)
{
// We don't yet support multireg locals.
assert((lclNode->GetLclNum() == varNum) && !lclNode->IsMultiReg());
assert(lclNode->GetLclNum() == varNum);
lclNode->SetRegNum(reg);
}
//-----------------------------------------------------------------------------
// LinearScan::resolveLocalRef // LinearScan::resolveLocalRef
// Description: // Description:
// Update the graph for a local reference. // Update the graph for a local reference.
@ -6318,7 +6340,7 @@ void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval, Regi
// NICE: Consider tracking whether an Interval is always in the same location (register/stack) // NICE: Consider tracking whether an Interval is always in the same location (register/stack)
// in which case it will require no resolution. // in which case it will require no resolution.
// //
void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPosition* currentRefPosition) void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, RefPosition* currentRefPosition)
{ {
assert((block == nullptr) == (treeNode == nullptr)); assert((block == nullptr) == (treeNode == nullptr));
assert(enregisterLocalVars); assert(enregisterLocalVars);
@ -6339,11 +6361,11 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
{ {
if (currentRefPosition->lastUse) if (currentRefPosition->lastUse)
{ {
treeNode->gtFlags |= GTF_VAR_DEATH; treeNode->SetLastUse(currentRefPosition->getMultiRegIdx());
} }
else else
{ {
treeNode->gtFlags &= ~GTF_VAR_DEATH; treeNode->ClearLastUse(currentRefPosition->getMultiRegIdx());
} }
if ((currentRefPosition->registerAssignment != RBM_NONE) && (interval->physReg == REG_NA) && if ((currentRefPosition->registerAssignment != RBM_NONE) && (interval->physReg == REG_NA) &&
@ -6354,7 +6376,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
// during resolution. In this case we're better off making it contained. // during resolution. In this case we're better off making it contained.
assert(inVarToRegMaps[curBBNum][varDsc->lvVarIndex] == REG_STK); assert(inVarToRegMaps[curBBNum][varDsc->lvVarIndex] == REG_STK);
currentRefPosition->registerAssignment = RBM_NONE; currentRefPosition->registerAssignment = RBM_NONE;
treeNode->SetRegNum(REG_NA); writeLocalReg(treeNode->AsLclVar(), interval->varNum, REG_NA);
} }
} }
@ -6445,9 +6467,11 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
// //
// Note that varDsc->GetRegNum() is already to REG_STK above. // Note that varDsc->GetRegNum() is already to REG_STK above.
interval->physReg = REG_NA; interval->physReg = REG_NA;
treeNode->SetRegNum(REG_NA); writeLocalReg(treeNode->AsLclVar(), interval->varNum, REG_NA);
treeNode->gtFlags &= ~GTF_SPILLED; treeNode->gtFlags &= ~GTF_SPILLED;
treeNode->SetContained(); treeNode->SetContained();
// We don't support RegOptional for multi-reg localvars.
assert(!treeNode->IsMultiReg());
} }
else else
{ {
@ -6470,7 +6494,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
interval->physReg = REG_NA; interval->physReg = REG_NA;
if (treeNode != nullptr) if (treeNode != nullptr)
{ {
treeNode->SetRegNum(REG_NA); writeLocalReg(treeNode->AsLclVar(), interval->varNum, REG_NA);
} }
} }
else // Not reload and Not pure-def that's spillAfter else // Not reload and Not pure-def that's spillAfter
@ -6489,7 +6513,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPositi
// But for copyReg, the homeReg remains unchanged. // But for copyReg, the homeReg remains unchanged.
assert(treeNode != nullptr); assert(treeNode != nullptr);
treeNode->SetRegNum(interval->physReg); writeLocalReg(treeNode->AsLclVar(), interval->varNum, interval->physReg);
if (currentRefPosition->copyReg) if (currentRefPosition->copyReg)
{ {
@ -7345,9 +7369,9 @@ void LinearScan::resolveRegisters()
{ {
writeRegisters(currentRefPosition, treeNode); writeRegisters(currentRefPosition, treeNode);
if (treeNode->IsLocal() && currentRefPosition->getInterval()->isLocalVar) if (treeNode->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && currentRefPosition->getInterval()->isLocalVar)
{ {
resolveLocalRef(block, treeNode, currentRefPosition); resolveLocalRef(block, treeNode->AsLclVar(), currentRefPosition);
} }
// Mark spill locations on temps // Mark spill locations on temps
@ -7370,7 +7394,7 @@ void LinearScan::resolveRegisters()
treeNode->ResetReuseRegVal(); treeNode->ResetReuseRegVal();
} }
// In case of multi-reg call node, also set spill flag on the // In case of multi-reg node, also set spill flag on the
// register specified by multi-reg index of current RefPosition. // register specified by multi-reg index of current RefPosition.
// Note that the spill flag on treeNode indicates that one or // Note that the spill flag on treeNode indicates that one or
// more its allocated registers are in that state. // more its allocated registers are in that state.
@ -9305,7 +9329,7 @@ void Interval::dump()
} }
if (isStructField) if (isStructField)
{ {
printf(" (struct)"); printf(" (field)");
} }
if (isPromotedStruct) if (isPromotedStruct)
{ {
@ -9495,7 +9519,7 @@ void LinearScan::lsraGetOperandString(GenTree* tree,
void LinearScan::lsraDispNode(GenTree* tree, LsraTupleDumpMode mode, bool hasDest) void LinearScan::lsraDispNode(GenTree* tree, LsraTupleDumpMode mode, bool hasDest)
{ {
Compiler* compiler = JitTls::GetCompiler(); Compiler* compiler = JitTls::GetCompiler();
const unsigned operandStringLength = 16; const unsigned operandStringLength = 6 * MAX_MULTIREG_COUNT + 1;
char operandString[operandStringLength]; char operandString[operandStringLength];
const char* emptyDestOperand = " "; const char* emptyDestOperand = " ";
char spillChar = ' '; char spillChar = ' ';
@ -9635,7 +9659,7 @@ void LinearScan::TupleStyleDump(LsraTupleDumpMode mode)
{ {
BasicBlock* block; BasicBlock* block;
LsraLocation currentLoc = 1; // 0 is the entry LsraLocation currentLoc = 1; // 0 is the entry
const unsigned operandStringLength = 16; const unsigned operandStringLength = 6 * MAX_MULTIREG_COUNT + 1;
char operandString[operandStringLength]; char operandString[operandStringLength];
// currentRefPosition is not used for LSRA_DUMP_PRE // currentRefPosition is not used for LSRA_DUMP_PRE
@ -10799,6 +10823,7 @@ void LinearScan::verifyFinalAllocation()
regRecord->assignedInterval = interval; regRecord->assignedInterval = interval;
if (VERBOSE) if (VERBOSE)
{ {
dumpEmptyRefPosition();
printf("Move %-4s ", getRegName(regRecord->regNum)); printf("Move %-4s ", getRegName(regRecord->regNum));
} }
} }

8
src/coreclr/src/jit/lsra.h
View file

@ -965,8 +965,8 @@ private:
#ifdef DEBUG #ifdef DEBUG
void checkLastUses(BasicBlock* block); void checkLastUses(BasicBlock* block);
static int ComputeOperandDstCount(GenTree* operand); int ComputeOperandDstCount(GenTree* operand);
static int ComputeAvailableSrcCount(GenTree* node); int ComputeAvailableSrcCount(GenTree* node);
#endif // DEBUG #endif // DEBUG
void setFrameType(); void setFrameType();
@ -1090,7 +1090,8 @@ private:
RefPosition* buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskTP internalCands = RBM_NONE); RefPosition* buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskTP internalCands = RBM_NONE);
void buildInternalRegisterUses(); void buildInternalRegisterUses();
void resolveLocalRef(BasicBlock* block, GenTree* treeNode, RefPosition* currentRefPosition); void writeLocalReg(GenTreeLclVar* lclNode, unsigned varNum, regNumber reg);
void resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, RefPosition* currentRefPosition);
void insertMove(BasicBlock* block, GenTree* insertionPoint, unsigned lclNum, regNumber inReg, regNumber outReg); void insertMove(BasicBlock* block, GenTree* insertionPoint, unsigned lclNum, regNumber inReg, regNumber outReg);
@ -1575,6 +1576,7 @@ private:
int BuildBlockStore(GenTreeBlk* blkNode); int BuildBlockStore(GenTreeBlk* blkNode);
int BuildModDiv(GenTree* tree); int BuildModDiv(GenTree* tree);
int BuildIntrinsic(GenTree* tree); int BuildIntrinsic(GenTree* tree);
void BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, LclVarDsc* varDsc, RefPosition* singleUseRef, int index);
int BuildStoreLoc(GenTreeLclVarCommon* tree); int BuildStoreLoc(GenTreeLclVarCommon* tree);
int BuildIndir(GenTreeIndir* indirTree); int BuildIndir(GenTreeIndir* indirTree);
int BuildGCWriteBarrier(GenTree* tree); int BuildGCWriteBarrier(GenTree* tree);

2
src/coreclr/src/jit/morph.cpp
View file

@ -1950,8 +1950,6 @@ GenTree* Compiler::fgMakeTmpArgNode(fgArgTabEntry* curArgTabEntry)
assert(varTypeIsStruct(type)); assert(varTypeIsStruct(type));
if (lvaIsMultiregStruct(varDsc, curArgTabEntry->IsVararg())) if (lvaIsMultiregStruct(varDsc, curArgTabEntry->IsVararg()))
{ {
// ToDo-ARM64: Consider using: arg->ChangeOper(GT_LCL_FLD);
// as that is how UNIX_AMD64_ABI works.
// We will create a GT_OBJ for the argument below. // We will create a GT_OBJ for the argument below.
// This will be passed by value in two registers. // This will be passed by value in two registers.
assert(addrNode != nullptr); assert(addrNode != nullptr);

2
src/coreclr/src/jit/rationalize.cpp
View file

@ -295,7 +295,7 @@ static void RewriteAssignmentIntoStoreLclCore(GenTreeOp* assignment,
store->AsLclFld()->SetFieldSeq(var->AsLclFld()->GetFieldSeq()); store->AsLclFld()->SetFieldSeq(var->AsLclFld()->GetFieldSeq());
} }
copyFlags(store, var, GTF_LIVENESS_MASK); copyFlags(store, var, (GTF_LIVENESS_MASK | GTF_VAR_MULTIREG));
store->gtFlags &= ~GTF_REVERSE_OPS; store->gtFlags &= ~GTF_REVERSE_OPS;
store->gtType = var->TypeGet(); store->gtType = var->TypeGet();

38
src/coreclr/src/jit/regset.cpp
View file

@ -285,17 +285,16 @@ RegSet::SpillDsc* RegSet::rsGetSpillInfo(GenTree* tree, regNumber reg, SpillDsc*
// Return Value: // Return Value:
// None. // None.
// //
// Assumption: // Notes:
// RyuJIT backend specific: in case of multi-reg call nodes, GTF_SPILL // For multi-reg nodes, only the spill flag associated with this reg is cleared.
// flag associated with the reg that is being spilled is cleared. The // The spill flag on the node should be cleared by the caller of this method.
// caller of this method is expected to clear GTF_SPILL flag on call
// node after all of its registers marked for spilling are spilled.
// //
void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */) void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
{ {
assert(tree != nullptr); assert(tree != nullptr);
GenTreeCall* call = nullptr; GenTreeCall* call = nullptr;
GenTreeLclVar* lcl = nullptr;
var_types treeType; var_types treeType;
#if defined(TARGET_ARM) #if defined(TARGET_ARM)
GenTreePutArgSplit* splitArg = nullptr; GenTreePutArgSplit* splitArg = nullptr;
@ -320,6 +319,12 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
treeType = multiReg->GetRegType(regIdx); treeType = multiReg->GetRegType(regIdx);
} }
#endif // TARGET_ARM #endif // TARGET_ARM
else if (tree->IsMultiRegLclVar())
{
GenTreeLclVar* lcl = tree->AsLclVar();
LclVarDsc* varDsc = m_rsCompiler->lvaGetDesc(lcl->GetLclNum());
treeType = varDsc->TypeGet();
}
else else
{ {
treeType = tree->TypeGet(); treeType = tree->TypeGet();
@ -345,9 +350,8 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
// vars should be handled elsewhere, and to prevent // vars should be handled elsewhere, and to prevent
// spilling twice clear GTF_SPILL flag on tree node. // spilling twice clear GTF_SPILL flag on tree node.
// //
// In case of multi-reg call nodes only the spill flag // In case of multi-reg nodes, only the spill flag associated with this reg is cleared.
// associated with the reg is cleared. Spill flag on // The spill flag on the node should be cleared by the caller of this method.
// call node should be cleared by the caller of this method.
assert((tree->gtFlags & GTF_SPILL) != 0); assert((tree->gtFlags & GTF_SPILL) != 0);
unsigned regFlags = 0; unsigned regFlags = 0;
@ -371,6 +375,12 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
regFlags &= ~GTF_SPILL; regFlags &= ~GTF_SPILL;
} }
#endif // TARGET_ARM #endif // TARGET_ARM
else if (lcl != nullptr)
{
regFlags = lcl->GetRegSpillFlagByIdx(regIdx);
assert((regFlags & GTF_SPILL) != 0);
regFlags &= ~GTF_SPILL;
}
else else
{ {
assert(!varTypeIsMultiReg(tree)); assert(!varTypeIsMultiReg(tree));
@ -446,6 +456,11 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */)
multiReg->SetRegSpillFlagByIdx(regFlags, regIdx); multiReg->SetRegSpillFlagByIdx(regFlags, regIdx);
} }
#endif // TARGET_ARM #endif // TARGET_ARM
else if (lcl != nullptr)
{
regFlags |= GTF_SPILLED;
lcl->SetRegSpillFlagByIdx(regFlags, regIdx);
}
} }
#if defined(TARGET_X86) #if defined(TARGET_X86)
@ -565,6 +580,13 @@ TempDsc* RegSet::rsUnspillInPlace(GenTree* tree, regNumber oldReg, unsigned regI
multiReg->SetRegSpillFlagByIdx(flags, regIdx); multiReg->SetRegSpillFlagByIdx(flags, regIdx);
} }
#endif // TARGET_ARM #endif // TARGET_ARM
else if (tree->IsMultiRegLclVar())
{
GenTreeLclVar* lcl = tree->AsLclVar();
unsigned flags = lcl->GetRegSpillFlagByIdx(regIdx);
flags &= ~GTF_SPILLED;
lcl->SetRegSpillFlagByIdx(flags, regIdx);
}
else else
{ {
tree->gtFlags &= ~GTF_SPILLED; tree->gtFlags &= ~GTF_SPILLED;

4
src/coreclr/src/jit/ssabuilder.cpp
View file

@ -1650,6 +1650,10 @@ bool SsaBuilder::IncludeInSsa(unsigned lclNum)
// //
return false; return false;
} }
else if (varDsc->lvIsStructField && m_pCompiler->lvaGetDesc(varDsc->lvParentLcl)->lvIsMultiRegRet)
{
return false;
}
// otherwise this variable is included in SSA // otherwise this variable is included in SSA
return true; return true;
} }

60
src/coreclr/src/jit/treelifeupdater.cpp
View file

@ -74,9 +74,27 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
} }
// if it's a partial definition then variable "x" must have had a previous, original, site to be born. // if it's a partial definition then variable "x" must have had a previous, original, site to be born.
bool isBorn = ((lclVarTree->gtFlags & GTF_VAR_DEF) != 0 && (lclVarTree->gtFlags & GTF_VAR_USEASG) == 0); bool isBorn;
bool isDying = ((lclVarTree->gtFlags & GTF_VAR_DEATH) != 0); bool isDying;
bool spill = ((lclVarTree->gtFlags & GTF_SPILL) != 0); bool spill;
bool isMultiRegLocal = lclVarTree->IsMultiRegLclVar();
if (isMultiRegLocal)
{
assert((tree->gtFlags & GTF_VAR_USEASG) == 0);
isBorn = ((tree->gtFlags & GTF_VAR_DEF) != 0);
// Note that for multireg locals we can have definitions for which some of those are last uses.
// We don't want to add those to the varDeltaSet because otherwise they will be added as newly
// live.
isDying = !isBorn && tree->AsLclVar()->HasLastUse();
// GTF_SPILL will be set if any registers need to be spilled.
spill = ((tree->gtFlags & GTF_SPILL) != 0);
}
else
{
isBorn = ((lclVarTree->gtFlags & GTF_VAR_DEF) != 0 && (lclVarTree->gtFlags & GTF_VAR_USEASG) == 0);
isDying = ((lclVarTree->gtFlags & GTF_VAR_DEATH) != 0);
spill = ((lclVarTree->gtFlags & GTF_SPILL) != 0);
}
// Since all tracked vars are register candidates, but not all are in registers at all times, // Since all tracked vars are register candidates, but not all are in registers at all times,
// we maintain two separate sets of variables - the total set of variables that are either // we maintain two separate sets of variables - the total set of variables that are either
@ -108,6 +126,10 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
} }
} }
} }
else if (ForCodeGen && lclVarTree->IsMultiRegLclVar())
{
assert(!"MultiRegLclVars not yet supported");
}
else if (varDsc->lvPromoted) else if (varDsc->lvPromoted)
{ {
// If hasDeadTrackedFieldVars is true, then, for a LDOBJ(ADDR(<promoted struct local>)), // If hasDeadTrackedFieldVars is true, then, for a LDOBJ(ADDR(<promoted struct local>)),
@ -126,15 +148,17 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
} }
} }
for (unsigned i = varDsc->lvFieldLclStart; i < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; ++i) unsigned firstFieldVarNum = varDsc->lvFieldLclStart;
for (unsigned i = 0; i < varDsc->lvFieldCnt; ++i)
{ {
LclVarDsc* fldVarDsc = &(compiler->lvaTable[i]); LclVarDsc* fldVarDsc = compiler->lvaGetDesc(firstFieldVarNum + i);
noway_assert(fldVarDsc->lvIsStructField); noway_assert(fldVarDsc->lvIsStructField);
if (fldVarDsc->lvTracked) if (fldVarDsc->lvTracked)
{ {
unsigned fldVarIndex = fldVarDsc->lvVarIndex; unsigned fldVarIndex = fldVarDsc->lvVarIndex;
bool isInReg = fldVarDsc->lvIsInReg(); // We should never see enregistered fields in a struct local unless
bool isInMemory = !isInReg || fldVarDsc->lvLiveInOutOfHndlr; // IsMultiRegLclVar() returns true, in which case we've handled this above.
assert(!fldVarDsc->lvIsInReg());
noway_assert(fldVarIndex < compiler->lvaTrackedCount); noway_assert(fldVarIndex < compiler->lvaTrackedCount);
if (!hasDeadTrackedFieldVars) if (!hasDeadTrackedFieldVars)
{ {
@ -143,38 +167,16 @@ void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
{ {
// We repeat this call here and below to avoid the VarSetOps::IsMember // We repeat this call here and below to avoid the VarSetOps::IsMember
// test in this, the common case, where we have no deadTrackedFieldVars. // test in this, the common case, where we have no deadTrackedFieldVars.
if (isInReg)
{
if (isBorn)
{
compiler->codeGen->genUpdateVarReg(fldVarDsc, tree);
}
compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(tree));
}
if (isInMemory)
{
VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex); VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex);
} }
} }
}
else if (ForCodeGen && VarSetOps::IsMember(compiler, varDeltaSet, fldVarIndex)) else if (ForCodeGen && VarSetOps::IsMember(compiler, varDeltaSet, fldVarIndex))
{
if (isInReg)
{
if (isBorn)
{
compiler->codeGen->genUpdateVarReg(fldVarDsc, tree);
}
compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(tree));
}
if (isInMemory)
{ {
VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex); VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex);
} }
} }
} }
} }
}
// First, update the live set // First, update the live set
if (isDying) if (isDying)