Lines Matching refs:VT
198 EVT VT = getOptimalMemOpType(Op, FuncAttributes); in findOptimalMemOpLowering() local
200 if (VT == MVT::Other) { in findOptimalMemOpLowering()
204 VT = MVT::i64; in findOptimalMemOpLowering()
206 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) && in findOptimalMemOpLowering()
207 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign())) in findOptimalMemOpLowering()
208 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); in findOptimalMemOpLowering()
209 assert(VT.isInteger()); in findOptimalMemOpLowering()
219 if (VT.bitsGT(LVT)) in findOptimalMemOpLowering()
220 VT = LVT; in findOptimalMemOpLowering()
226 unsigned VTSize = VT.getSizeInBits() / 8; in findOptimalMemOpLowering()
229 EVT NewVT = VT; in findOptimalMemOpLowering()
233 if (VT.isVector() || VT.isFloatingPoint()) { in findOptimalMemOpLowering()
234 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; in findOptimalMemOpLowering()
261 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), in findOptimalMemOpLowering()
266 VT = NewVT; in findOptimalMemOpLowering()
274 MemOps.push_back(VT); in findOptimalMemOpLowering()
283 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, in softenSetCCOperands() argument
289 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS, in softenSetCCOperands()
293 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, in softenSetCCOperands() argument
304 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) in softenSetCCOperands()
313 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : in softenSetCCOperands()
314 (VT == MVT::f64) ? RTLIB::OEQ_F64 : in softenSetCCOperands()
315 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128; in softenSetCCOperands()
319 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : in softenSetCCOperands()
320 (VT == MVT::f64) ? RTLIB::UNE_F64 : in softenSetCCOperands()
321 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128; in softenSetCCOperands()
325 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : in softenSetCCOperands()
326 (VT == MVT::f64) ? RTLIB::OGE_F64 : in softenSetCCOperands()
327 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128; in softenSetCCOperands()
331 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : in softenSetCCOperands()
332 (VT == MVT::f64) ? RTLIB::OLT_F64 : in softenSetCCOperands()
333 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128; in softenSetCCOperands()
337 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : in softenSetCCOperands()
338 (VT == MVT::f64) ? RTLIB::OLE_F64 : in softenSetCCOperands()
339 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128; in softenSetCCOperands()
343 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : in softenSetCCOperands()
344 (VT == MVT::f64) ? RTLIB::OGT_F64 : in softenSetCCOperands()
345 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128; in softenSetCCOperands()
351 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : in softenSetCCOperands()
352 (VT == MVT::f64) ? RTLIB::UO_F64 : in softenSetCCOperands()
353 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128; in softenSetCCOperands()
360 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : in softenSetCCOperands()
361 (VT == MVT::f64) ? RTLIB::UO_F64 : in softenSetCCOperands()
362 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128; in softenSetCCOperands()
363 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : in softenSetCCOperands()
364 (VT == MVT::f64) ? RTLIB::OEQ_F64 : in softenSetCCOperands()
365 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128; in softenSetCCOperands()
372 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : in softenSetCCOperands()
373 (VT == MVT::f64) ? RTLIB::OGE_F64 : in softenSetCCOperands()
374 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128; in softenSetCCOperands()
377 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : in softenSetCCOperands()
378 (VT == MVT::f64) ? RTLIB::OGT_F64 : in softenSetCCOperands()
379 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128; in softenSetCCOperands()
382 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : in softenSetCCOperands()
383 (VT == MVT::f64) ? RTLIB::OLE_F64 : in softenSetCCOperands()
384 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128; in softenSetCCOperands()
387 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : in softenSetCCOperands()
388 (VT == MVT::f64) ? RTLIB::OLT_F64 : in softenSetCCOperands()
389 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128; in softenSetCCOperands()
522 EVT VT = Op.getValueType(); in ShrinkDemandedConstant() local
523 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT); in ShrinkDemandedConstant()
524 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC); in ShrinkDemandedConstant()
538 EVT VT = Op.getValueType(); in ShrinkDemandedConstant() local
539 APInt DemandedElts = VT.isVector() in ShrinkDemandedConstant()
540 ? APInt::getAllOnesValue(VT.getVectorNumElements()) in ShrinkDemandedConstant()
612 EVT VT = Op.getValueType(); in SimplifyDemandedBits() local
617 if (VT.isScalableVector()) { in SimplifyDemandedBits()
623 APInt DemandedElts = VT.isVector() in SimplifyDemandedBits()
624 ? APInt::getAllOnesValue(VT.getVectorNumElements()) in SimplifyDemandedBits()
867 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBits() local
868 APInt DemandedElts = VT.isVector() in SimplifyMultipleUseDemandedBits()
869 ? APInt::getAllOnesValue(VT.getVectorNumElements()) in SimplifyMultipleUseDemandedBits()
935 EVT VT = Op.getValueType(); in SimplifyDemandedBits() local
949 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedBits()
961 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedBits()
1068 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub, in SimplifyDemandedBits()
1093 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, in SimplifyDemandedBits()
1169 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask); in SimplifyDemandedBits()
1204 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1227 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1240 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT)); in SimplifyDemandedBits()
1274 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1317 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1336 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1347 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT); in SimplifyDemandedBits()
1348 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1357 SDValue New = TLO.DAG.getNOT(dl, Op0, VT); in SimplifyDemandedBits()
1460 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1481 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1501 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1504 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1568 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1609 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1643 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
1650 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
1662 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1784 if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector()) in SimplifyDemandedBits()
1785 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
1806 EVT ShiftAmtTy = VT; in SimplifyDemandedBits()
1813 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
1883 VT.getSizeInBits() == SrcVT.getSizeInBits() && in SimplifyDemandedBits()
1885 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
1889 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
1890 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
1905 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
1921 VT.getSizeInBits() == SrcVT.getSizeInBits() && in SimplifyDemandedBits()
1923 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
1927 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
1928 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
1951 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
1952 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
1958 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
1972 VT.getSizeInBits() == SrcVT.getSizeInBits() && in SimplifyDemandedBits()
1974 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
1988 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2006 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2017 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT)) in SimplifyDemandedBits()
2037 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes())); in SimplifyDemandedBits()
2039 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2041 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2095 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx); in SimplifyDemandedBits()
2112 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() && in SimplifyDemandedBits()
2115 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT); in SimplifyDemandedBits()
2117 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 && in SimplifyDemandedBits()
2120 EVT Ty = OpVTLegal ? VT : MVT::i32; in SimplifyDemandedBits()
2126 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2128 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT); in SimplifyDemandedBits()
2130 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2218 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags); in SimplifyDemandedBits()
2236 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags); in SimplifyDemandedBits()
2250 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT); in SimplifyDemandedBits()
2255 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags); in SimplifyDemandedBits()
2285 if (VT.isInteger()) in SimplifyDemandedBits()
2286 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT)); in SimplifyDemandedBits()
2287 if (VT.isFloatingPoint()) in SimplifyDemandedBits()
2291 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT)); in SimplifyDemandedBits()
2321 EVT VT = BO.getValueType(); in getKnownUndefForVectorBinop() local
2322 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && in getKnownUndefForVectorBinop()
2325 EVT EltVT = VT.getVectorElementType(); in getKnownUndefForVectorBinop()
2326 unsigned NumElts = VT.getVectorNumElements(); in getKnownUndefForVectorBinop()
2368 EVT VT = Op.getValueType(); in SimplifyDemandedVectorElts() local
2372 assert(VT.isVector() && "Expected vector op"); in SimplifyDemandedVectorElts()
2377 if (VT.isScalableVector()) in SimplifyDemandedVectorElts()
2380 assert(VT.getVectorNumElements() == NumElts && in SimplifyDemandedVectorElts()
2396 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
2404 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in SimplifyDemandedVectorElts()
2415 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1); in SimplifyDemandedVectorElts()
2425 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
2553 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops)); in SimplifyDemandedVectorElts()
2601 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in SimplifyDemandedVectorElts()
2602 TLO.DAG.getUNDEF(VT), Sub, in SimplifyDemandedVectorElts()
2621 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, in SimplifyDemandedVectorElts()
2649 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, in SimplifyDemandedVectorElts()
2763 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1), in SimplifyDemandedVectorElts()
2805 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedVectorElts()
2811 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
2915 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
2939 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3039 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, in buildLegalVectorShuffle() argument
3042 bool LegalMask = isShuffleMaskLegal(Mask, VT); in buildLegalVectorShuffle()
3046 LegalMask = isShuffleMaskLegal(Mask, VT); in buildLegalVectorShuffle()
3052 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask); in buildLegalVectorShuffle()
3146 bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT, in isExtendedTrueVal() argument
3148 if (VT == MVT::i1) in isExtendedTrueVal()
3151 TargetLowering::BooleanContent Cnt = getBooleanContents(VT); in isExtendedTrueVal()
3166 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithAnd() argument
3202 return DAG.getSetCC(DL, VT, N0, Zero, Cond); in foldSetCCWithAnd()
3219 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond); in foldSetCCWithAnd()
3384 EVT VT = X.getValueType(); in optimizeSetCCByHoistingAndByConstFromLogicalShift() local
3388 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3389 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3397 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithBinOp() argument
3413 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond); in foldSetCCWithBinOp()
3421 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond); in foldSetCCWithBinOp()
3434 return DAG.getSetCC(DL, VT, X, YShl1, Cond); in foldSetCCWithBinOp()
3437 static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, in simplifySetCCWithCTPOP() argument
3444 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && in simplifySetCCWithCTPOP()
3456 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT)) in simplifySetCCWithCTPOP()
3477 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC); in simplifySetCCWithCTPOP()
3483 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT)) in simplifySetCCWithCTPOP()
3496 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond); in simplifySetCCWithCTPOP()
3497 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond); in simplifySetCCWithCTPOP()
3499 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS); in simplifySetCCWithCTPOP()
3507 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
3516 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl)) in SimplifySetCC()
3528 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
3539 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
3545 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG)) in SimplifySetCC()
3567 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero, in SimplifySetCC()
3628 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1), in SimplifySetCC()
3631 return DAG.getSetCC(dl, VT, Trunc, C, Cond); in SimplifySetCC()
3645 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 && in SimplifySetCC()
3660 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), in SimplifySetCC()
3716 return DAG.getSetCC(dl, VT, in SimplifySetCC()
3737 return DAG.getConstant(0, dl, VT); in SimplifySetCC()
3741 return DAG.getConstant(1, dl, VT); in SimplifySetCC()
3745 return DAG.getConstant(C1.isNegative(), dl, VT); in SimplifySetCC()
3749 return DAG.getConstant(C1.isNonNegative(), dl, VT); in SimplifySetCC()
3772 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType()); in SimplifySetCC()
3791 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
3801 return DAG.getSetCC(dl, VT, ZextOp, in SimplifySetCC()
3807 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) && in SimplifySetCC()
3813 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
3819 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in SimplifySetCC()
3846 return DAG.getSetCC(dl, VT, Val, N1, in SimplifySetCC()
3867 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond); in SimplifySetCC()
3872 if (Op0.getValueType().bitsGT(VT)) in SimplifySetCC()
3873 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
3874 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
3875 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
3876 else if (Op0.getValueType().bitsLT(VT)) in SimplifySetCC()
3877 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
3878 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
3879 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
3881 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
3887 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
3902 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
3906 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
3929 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
3931 if (!VT.isVector()) { // TODO: Support this for vectors. in SimplifySetCC()
3936 isCondCodeLegal(NewCC, VT.getSimpleVT())) && in SimplifySetCC()
3939 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
3949 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
3952 if (!VT.isVector()) { // TODO: Support this for vectors. in SimplifySetCC()
3956 isCondCodeLegal(NewCC, VT.getSimpleVT())) && in SimplifySetCC()
3959 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
3968 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false in SimplifySetCC()
3971 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
3974 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
3978 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
3986 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false in SimplifySetCC()
3989 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
3992 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
3996 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4006 VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4053 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond); in SimplifySetCC()
4074 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
4079 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4087 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4104 if ((VT.getSizeInBits() == 1 || in SimplifySetCC()
4107 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) && in SimplifySetCC()
4117 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
4127 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
4150 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond); in SimplifySetCC()
4178 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond); in SimplifySetCC()
4193 return DAG.getSetCC(dl, VT, N0, N0, Cond); in SimplifySetCC()
4201 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond); in SimplifySetCC()
4223 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
4237 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
4239 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
4246 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
4256 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond); in SimplifySetCC()
4258 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond); in SimplifySetCC()
4262 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0), in SimplifySetCC()
4265 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1), in SimplifySetCC()
4278 return DAG.getSetCC(dl, VT, N0.getOperand(0), in SimplifySetCC()
4290 DAG.getSetCC(dl, VT, N0.getOperand(0), in SimplifySetCC()
4301 DAG.getSetCC(dl, VT, N0.getOperand(1), in SimplifySetCC()
4319 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
4325 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI)) in SimplifySetCC()
4328 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
4339 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) { in SimplifySetCC()
4341 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4344 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4391 if (VT.getScalarType() != MVT::i1) { in SimplifySetCC()
4396 N0 = DAG.getNode(ExtendCode, dl, VT, N0); in SimplifySetCC()
4590 MVT VT) const { in getRegForInlineAsmConstraint()
4616 if (RI->isTypeLegalForClass(*RC, VT)) in getRegForInlineAsmConstraint()
5044 EVT VT = N->getValueType(0); in BuildExactSDIV() local
5045 EVT SVT = VT.getScalarType(); in BuildExactSDIV()
5046 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in BuildExactSDIV()
5078 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildExactSDIV()
5084 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildExactSDIV()
5098 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
5102 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
5123 EVT VT = N->getValueType(0); in BuildSDIV() local
5124 EVT SVT = VT.getScalarType(); in BuildSDIV()
5125 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in BuildSDIV()
5127 unsigned EltBits = VT.getScalarSizeInBits(); in BuildSDIV()
5132 if (!isTypeLegal(VT)) { in BuildSDIV()
5134 if (VT.isVector() || !VT.isSimple()) in BuildSDIV()
5139 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) in BuildSDIV()
5142 MulVT = getTypeToTransformTo(*DAG.getContext(), VT); in BuildSDIV()
5193 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildSDIV()
5194 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildSDIV()
5196 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks); in BuildSDIV()
5202 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildSDIV()
5203 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildSDIV()
5205 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]); in BuildSDIV()
5219 if (!isTypeLegal(VT)) { in BuildSDIV()
5225 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
5228 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) in BuildSDIV()
5229 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
5230 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { in BuildSDIV()
5232 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
5245 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
5247 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
5251 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
5256 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
5258 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
5260 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
5271 EVT VT = N->getValueType(0); in BuildUDIV() local
5272 EVT SVT = VT.getScalarType(); in BuildUDIV()
5273 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in BuildUDIV()
5275 unsigned EltBits = VT.getScalarSizeInBits(); in BuildUDIV()
5280 if (!isTypeLegal(VT)) { in BuildUDIV()
5282 if (VT.isVector() || !VT.isSimple()) in BuildUDIV()
5287 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) in BuildUDIV()
5290 MulVT = getTypeToTransformTo(*DAG.getContext(), VT); in BuildUDIV()
5351 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildUDIV()
5352 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors); in BuildUDIV()
5359 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildUDIV()
5360 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]); in BuildUDIV()
5370 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
5377 if (!isTypeLegal(VT)) { in BuildUDIV()
5383 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
5386 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) in BuildUDIV()
5387 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
5388 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { in BuildUDIV()
5390 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
5404 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
5409 if (VT.isVector()) in BuildUDIV()
5412 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
5416 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
5420 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
5423 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in BuildUDIV()
5425 SDValue One = DAG.getConstant(1, dl, VT); in BuildUDIV()
5427 return DAG.getSelect(dl, VT, IsOne, N0, Q); in BuildUDIV()
5494 EVT VT = REMNode.getValueType(); in prepareUREMEqFold() local
5495 EVT SVT = VT.getScalarType(); in prepareUREMEqFold()
5496 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize()); in prepareUREMEqFold()
5500 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareUREMEqFold()
5620 PVal = DAG.getBuildVector(VT, DL, PAmts); in prepareUREMEqFold()
5622 QVal = DAG.getBuildVector(VT, DL, QAmts); in prepareUREMEqFold()
5627 PVal = DAG.getSplatVector(VT, DL, PAmts[0]); in prepareUREMEqFold()
5629 QVal = DAG.getSplatVector(VT, DL, QAmts[0]); in prepareUREMEqFold()
5637 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) in prepareUREMEqFold()
5641 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode); in prepareUREMEqFold()
5645 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareUREMEqFold()
5652 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareUREMEqFold()
5655 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareUREMEqFold()
5669 assert(VT.isVector() && "Can/should only get here for vectors."); in prepareUREMEqFold()
5743 EVT VT = REMNode.getValueType(); in prepareSREMEqFold() local
5744 EVT SVT = VT.getScalarType(); in prepareSREMEqFold()
5745 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize()); in prepareSREMEqFold()
5750 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareSREMEqFold()
5878 PVal = DAG.getBuildVector(VT, DL, PAmts); in prepareSREMEqFold()
5879 AVal = DAG.getBuildVector(VT, DL, AAmts); in prepareSREMEqFold()
5881 QVal = DAG.getBuildVector(VT, DL, QAmts); in prepareSREMEqFold()
5887 PVal = DAG.getSplatVector(VT, DL, PAmts[0]); in prepareSREMEqFold()
5888 AVal = DAG.getSplatVector(VT, DL, AAmts[0]); in prepareSREMEqFold()
5890 QVal = DAG.getSplatVector(VT, DL, QAmts[0]); in prepareSREMEqFold()
5900 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareSREMEqFold()
5905 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) in prepareSREMEqFold()
5909 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal); in prepareSREMEqFold()
5917 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareSREMEqFold()
5920 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareSREMEqFold()
5936 assert(VT.isVector() && "Can/should only get here for vectors."); in prepareSREMEqFold()
5941 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) || in prepareSREMEqFold()
5942 !isOperationLegalOrCustom(ISD::AND, VT) || in prepareSREMEqFold()
5943 !isOperationLegalOrCustom(Cond, VT) || in prepareSREMEqFold()
5950 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
5952 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
5954 DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
5961 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax); in prepareSREMEqFold()
5990 EVT VT = Op.getValueType(); in getSqrtInputTest() local
5991 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSqrtInputTest()
5992 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in getSqrtInputTest()
5999 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT); in getSqrtInputTest()
6001 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT); in getSqrtInputTest()
6002 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in getSqrtInputTest()
6027 EVT VT = Op.getValueType(); in getNegatedExpression() local
6033 isFPExtFree(VT, Op.getOperand(0).getValueType()); in getNegatedExpression()
6055 isOperationLegal(ISD::ConstantFP, VT) || in getNegatedExpression()
6056 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT, in getNegatedExpression()
6064 SDValue CFP = DAG.getConstantFP(V, DL, VT); in getNegatedExpression()
6081 (isOperationLegal(ISD::ConstantFP, VT) && in getNegatedExpression()
6082 isOperationLegal(ISD::BUILD_VECTOR, VT)) || in getNegatedExpression()
6085 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT, in getNegatedExpression()
6103 return DAG.getBuildVector(VT, DL, Ops); in getNegatedExpression()
6110 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT)) in getNegatedExpression()
6133 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags); in getNegatedExpression()
6142 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags); in getNegatedExpression()
6164 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags); in getNegatedExpression()
6189 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags); in getNegatedExpression()
6203 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags); in getNegatedExpression()
6245 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags); in getNegatedExpression()
6254 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags); in getNegatedExpression()
6266 return DAG.getNode(Opcode, DL, VT, NegV); in getNegatedExpression()
6271 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1)); in getNegatedExpression()
6282 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, in expandMUL_LOHI() argument
6303 unsigned OuterBitSize = VT.getScalarSizeInBits(); in expandMUL_LOHI()
6353 if (!VT.isVector() && Opcode == ISD::MUL && in expandMUL_LOHI()
6366 EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandMUL_LOHI()
6376 isOperationLegalOrCustom(ISD::SRL, VT) && in expandMUL_LOHI()
6378 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
6380 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
6403 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
6404 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
6405 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
6406 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
6409 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
6415 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
6421 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandMUL_LOHI()
6423 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) && in expandMUL_LOHI()
6424 isOperationLegalOrCustom(ISD::ADDE, VT)); in expandMUL_LOHI()
6426 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
6429 Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
6434 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
6446 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
6449 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
6450 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
6453 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
6454 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
6459 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
6490 EVT VT = Node->getValueType(0); in expandFunnelShift() local
6492 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandFunnelShift()
6493 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandFunnelShift()
6494 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandFunnelShift()
6495 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandFunnelShift()
6502 unsigned BW = VT.getScalarSizeInBits(); in expandFunnelShift()
6510 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) && in expandFunnelShift()
6511 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) { in expandFunnelShift()
6516 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z); in expandFunnelShift()
6522 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One); in expandFunnelShift()
6523 X = DAG.getNode(ISD::SRL, DL, VT, X, One); in expandFunnelShift()
6525 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One); in expandFunnelShift()
6526 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One); in expandFunnelShift()
6530 Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z); in expandFunnelShift()
6543 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt); in expandFunnelShift()
6544 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt); in expandFunnelShift()
6562 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt); in expandFunnelShift()
6563 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One); in expandFunnelShift()
6564 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt); in expandFunnelShift()
6566 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One); in expandFunnelShift()
6567 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt); in expandFunnelShift()
6568 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt); in expandFunnelShift()
6571 Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in expandFunnelShift()
6578 EVT VT = Node->getValueType(0); in expandROT() local
6579 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in expandROT()
6590 if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) { in expandROT()
6592 Result = DAG.getNode(RevRot, DL, VT, Op0, Sub); in expandROT()
6596 if (!AllowVectorOps && VT.isVector() && in expandROT()
6597 (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandROT()
6598 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandROT()
6599 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandROT()
6600 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) || in expandROT()
6601 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandROT()
6614 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt); in expandROT()
6616 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt); in expandROT()
6622 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt); in expandROT()
6626 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt); in expandROT()
6628 Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal); in expandROT()
6635 EVT VT = Node->getValueType(0); in expandShiftParts() local
6636 unsigned VTBits = VT.getScalarSizeInBits(); in expandShiftParts()
6654 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
6656 : DAG.getConstant(0, dl, VT); in expandShiftParts()
6660 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
6661 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
6663 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
6664 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
6676 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
6677 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
6679 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
6680 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
6913 EVT VT = Node->getValueType(0); in expandFMINNUM_FMAXNUM() local
6915 if (VT.isScalableVector()) in expandFMINNUM_FMAXNUM()
6919 if (isOperationLegalOrCustom(NewOp, VT)) { in expandFMINNUM_FMAXNUM()
6927 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
6931 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
6936 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags()); in expandFMINNUM_FMAXNUM()
6944 if (isOperationLegalOrCustom(IEEE2018Op, VT)) { in expandFMINNUM_FMAXNUM()
6945 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0), in expandFMINNUM_FMAXNUM()
6976 EVT VT = Node->getValueType(0); in expandCTPOP() local
6977 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandCTPOP()
6979 unsigned Len = VT.getScalarSizeInBits(); in expandCTPOP()
6980 assert(VT.isInteger() && "CTPOP not implemented for this type."); in expandCTPOP()
6987 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) || in expandCTPOP()
6988 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTPOP()
6989 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTPOP()
6990 (Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) || in expandCTPOP()
6991 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandCTPOP()
6997 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandCTPOP()
6999 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandCTPOP()
7001 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandCTPOP()
7003 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandCTPOP()
7006 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
7007 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7008 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7012 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
7013 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7014 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7018 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7019 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
7020 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7026 DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), in expandCTPOP()
7036 EVT VT = Node->getValueType(0); in expandCTLZ() local
7037 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandCTLZ()
7039 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandCTLZ()
7043 isOperationLegalOrCustom(ISD::CTLZ, VT)) { in expandCTLZ()
7044 Result = DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
7049 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) { in expandCTLZ()
7051 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCTLZ()
7052 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
7053 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTLZ()
7055 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero, in expandCTLZ()
7056 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ); in expandCTLZ()
7061 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) || in expandCTLZ()
7062 !isOperationLegalOrCustom(ISD::CTPOP, VT) || in expandCTLZ()
7063 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTLZ()
7064 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandCTLZ()
7078 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
7079 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
7081 Op = DAG.getNOT(dl, Op, VT); in expandCTLZ()
7082 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
7089 EVT VT = Node->getValueType(0); in expandCTTZ() local
7091 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandCTTZ()
7095 isOperationLegalOrCustom(ISD::CTTZ, VT)) { in expandCTTZ()
7096 Result = DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
7101 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) { in expandCTTZ()
7103 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCTTZ()
7104 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
7105 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTTZ()
7107 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero, in expandCTTZ()
7108 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ); in expandCTTZ()
7113 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) || in expandCTTZ()
7114 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTTZ()
7115 !isOperationLegalOrCustom(ISD::CTLZ, VT)) || in expandCTTZ()
7116 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTTZ()
7117 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) || in expandCTTZ()
7118 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandCTTZ()
7126 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
7127 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
7130 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) { in expandCTTZ()
7132 DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
7133 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
7137 Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
7144 EVT VT = N->getValueType(0); in expandABS() local
7145 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandABS()
7149 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
7150 isOperationLegal(ISD::SMAX, VT)) { in expandABS()
7151 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
7152 Result = DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
7153 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
7158 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
7159 isOperationLegal(ISD::UMIN, VT)) { in expandABS()
7160 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
7161 Result = DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
7162 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
7167 if (IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
7168 isOperationLegal(ISD::SMIN, VT)) { in expandABS()
7169 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
7170 Result = DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
7171 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
7176 if (VT.isVector() && in expandABS()
7177 (!isOperationLegalOrCustom(ISD::SRA, VT) || in expandABS()
7178 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) || in expandABS()
7179 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) || in expandABS()
7180 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandABS()
7184 DAG.getNode(ISD::SRA, dl, VT, Op, in expandABS()
7185 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT)); in expandABS()
7187 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift); in expandABS()
7188 Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift); in expandABS()
7191 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
7192 Result = DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
7199 EVT VT = N->getValueType(0); in expandBSWAP() local
7202 if (!VT.isSimple()) in expandBSWAP()
7205 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandBSWAP()
7207 switch (VT.getSimpleVT().getScalarType().SimpleTy) { in expandBSWAP()
7212 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
7214 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
7215 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
7216 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
7217 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
7218 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
7219 DAG.getConstant(0xFF0000, dl, VT)); in expandBSWAP()
7220 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
7221 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
7222 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
7223 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
7225 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
7226 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
7227 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
7228 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
7229 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
7230 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
7231 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
7232 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
7233 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, in expandBSWAP()
7234 DAG.getConstant(255ULL<<48, dl, VT)); in expandBSWAP()
7235 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, in expandBSWAP()
7236 DAG.getConstant(255ULL<<40, dl, VT)); in expandBSWAP()
7237 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, in expandBSWAP()
7238 DAG.getConstant(255ULL<<32, dl, VT)); in expandBSWAP()
7239 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
7240 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
7241 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
7242 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
7243 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
7244 DAG.getConstant(255ULL<<8 , dl, VT)); in expandBSWAP()
7245 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
7246 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
7247 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
7248 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
7249 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
7250 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
7251 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
7257 EVT VT = N->getValueType(0); in expandBITREVERSE() local
7259 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandBITREVERSE()
7260 unsigned Sz = VT.getScalarSizeInBits(); in expandBITREVERSE()
7277 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
7280 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); in expandBITREVERSE()
7281 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); in expandBITREVERSE()
7282 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
7283 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
7284 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
7287 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); in expandBITREVERSE()
7288 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); in expandBITREVERSE()
7289 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
7290 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
7291 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
7294 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); in expandBITREVERSE()
7295 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); in expandBITREVERSE()
7296 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
7297 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
7298 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
7302 Tmp = DAG.getConstant(0, dl, VT); in expandBITREVERSE()
7306 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
7309 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
7313 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
7314 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
7494 EVT VT = LD->getValueType(0); in expandUnalignedLoad() local
7499 if (VT.isFloatingPoint() || VT.isVector()) { in expandUnalignedLoad()
7513 if (LoadedVT != VT) in expandUnalignedLoad()
7514 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in expandUnalignedLoad()
7515 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
7577 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, in expandUnalignedLoad()
7606 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
7611 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, in expandUnalignedLoad()
7616 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
7621 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
7631 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
7632 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
7647 EVT VT = Val.getValueType(); in expandUnalignedStore() local
7654 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in expandUnalignedStore()
7745 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
7925 EVT VT = Op.getOperand(0).getValueType(); in lowerCmpEqZeroToCtlzSrl() local
7927 if (VT.bitsLT(MVT::i32)) { in lowerCmpEqZeroToCtlzSrl()
7928 VT = MVT::i32; in lowerCmpEqZeroToCtlzSrl()
7929 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
7931 unsigned Log2b = Log2_32(VT.getSizeInBits()); in lowerCmpEqZeroToCtlzSrl()
7932 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
7933 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
7963 EVT VT = Op0.getValueType(); in expandIntMINMAX() local
7968 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) && in expandIntMINMAX()
7969 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
7970 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
7971 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1)); in expandIntMINMAX()
7975 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) && in expandIntMINMAX()
7976 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
7977 return DAG.getNode(ISD::ADD, DL, VT, Op0, in expandIntMINMAX()
7978 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0)); in expandIntMINMAX()
7993 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandIntMINMAX()
7996 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC); in expandIntMINMAX()
7997 return DAG.getSelect(DL, VT, Cond, Op0, Op1); in expandIntMINMAX()
8004 EVT VT = LHS.getValueType(); in expandAddSubSat() local
8007 assert(VT == RHS.getValueType() && "Expected operands to be the same type"); in expandAddSubSat()
8008 assert(VT.isInteger() && "Expected operands to be integers"); in expandAddSubSat()
8011 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) { in expandAddSubSat()
8012 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
8013 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
8017 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) { in expandAddSubSat()
8018 SDValue InvRHS = DAG.getNOT(dl, RHS, VT); in expandAddSubSat()
8019 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
8020 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
8044 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandAddSubSat()
8048 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandAddSubSat()
8049 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandAddSubSat()
8052 SDValue Zero = DAG.getConstant(0, dl, VT); in expandAddSubSat()
8053 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in expandAddSubSat()
8056 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandAddSubSat()
8058 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
8059 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
8062 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff); in expandAddSubSat()
8066 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandAddSubSat()
8068 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
8069 SDValue Not = DAG.getNOT(dl, OverflowMask, VT); in expandAddSubSat()
8070 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
8073 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff); in expandAddSubSat()
8080 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
8081 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandAddSubSat()
8083 Result = DAG.getSelect(dl, VT, SumNeg, SatMax, SatMin); in expandAddSubSat()
8084 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); in expandAddSubSat()
8092 EVT VT = LHS.getValueType(); in expandShlSat() local
8098 assert(VT == RHS.getValueType() && "Expected operands to be the same type"); in expandShlSat()
8099 assert(VT.isInteger() && "Expected operands to be integers"); in expandShlSat()
8103 unsigned BW = VT.getScalarSizeInBits(); in expandShlSat()
8104 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
8106 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
8110 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT); in expandShlSat()
8111 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT); in expandShlSat()
8112 SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT), in expandShlSat()
8115 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT); in expandShlSat()
8133 EVT VT = LHS.getValueType(); in expandFixedPointMul() local
8139 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFixedPointMul()
8140 unsigned VTSize = VT.getScalarSizeInBits(); in expandFixedPointMul()
8145 if (isOperationLegalOrCustom(ISD::MUL, VT)) in expandFixedPointMul()
8146 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
8147 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) { in expandFixedPointMul()
8149 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
8152 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
8156 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandFixedPointMul()
8157 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
8160 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
8162 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); in expandFixedPointMul()
8163 return DAG.getSelect(dl, VT, Overflow, Result, Product); in expandFixedPointMul()
8164 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) { in expandFixedPointMul()
8166 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
8171 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
8172 return DAG.getSelect(dl, VT, Overflow, SatMax, Product); in expandFixedPointMul()
8186 if (isOperationLegalOrCustom(LoHiOp, VT)) { in expandFixedPointMul()
8187 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS); in expandFixedPointMul()
8190 } else if (isOperationLegalOrCustom(HiOp, VT)) { in expandFixedPointMul()
8191 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
8192 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS); in expandFixedPointMul()
8193 } else if (VT.isVector()) { in expandFixedPointMul()
8208 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandFixedPointMul()
8209 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
8222 dl, VT); in expandFixedPointMul()
8224 DAG.getConstant(MaxVal, dl, VT), Result, in expandFixedPointMul()
8233 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT); in expandFixedPointMul()
8234 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT); in expandFixedPointMul()
8237 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
8242 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
8246 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result); in expandFixedPointMul()
8254 dl, VT); in expandFixedPointMul()
8260 dl, VT); in expandFixedPointMul()
8273 EVT VT = LHS.getValueType(); in expandFixedPointDiv() local
8276 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFixedPointDiv()
8305 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandFixedPointDiv()
8307 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
8310 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
8322 if (isTypeLegal(VT) && in expandFixedPointDiv()
8323 isOperationLegalOrCustom(ISD::SDIVREM, VT)) { in expandFixedPointDiv()
8325 DAG.getVTList(VT, VT), in expandFixedPointDiv()
8330 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
8332 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
8335 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointDiv()
8340 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
8341 DAG.getConstant(1, dl, VT)); in expandFixedPointDiv()
8342 Quot = DAG.getSelect(dl, VT, in expandFixedPointDiv()
8346 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
8424 EVT VT = Node->getValueType(0); in expandMULO() local
8425 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandMULO()
8437 EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandMULO()
8439 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
8442 dl, VT, Result, ShiftAmt), in expandMULO()
8448 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); in expandMULO()
8449 if (VT.isVector()) in expandMULO()
8451 VT.getVectorNumElements()); in expandMULO()
8458 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) { in expandMULO()
8459 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
8460 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); in expandMULO()
8461 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) { in expandMULO()
8462 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, in expandMULO()
8469 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
8470 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl, in expandMULO()
8472 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
8475 if (VT.isVector()) in expandMULO()
8498 unsigned LoSize = VT.getFixedSizeInBits(); in expandMULO()
8500 DAG.getNode(ISD::SRA, dl, VT, LHS, in expandMULO()
8504 DAG.getNode(ISD::SRA, dl, VT, RHS, in expandMULO()
8508 HiLHS = DAG.getConstant(0, dl, VT); in expandMULO()
8509 HiRHS = DAG.getConstant(0, dl, VT); in expandMULO()
8546 VT.getScalarSizeInBits() - 1, dl, in expandMULO()
8548 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
8552 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
8569 EVT VT = Op.getValueType(); in expandVecReduce() local
8571 if (VT.isScalableVector()) in expandVecReduce()
8576 if (VT.isPow2VectorType()) { in expandVecReduce()
8577 while (VT.getVectorNumElements() > 1) { in expandVecReduce()
8578 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in expandVecReduce()
8585 VT = HalfVT; in expandVecReduce()
8589 EVT EltVT = VT.getVectorElementType(); in expandVecReduce()
8590 unsigned NumElts = VT.getVectorNumElements(); in expandVecReduce()
8611 EVT VT = VecOp.getValueType(); in expandVecReduceSeq() local
8612 EVT EltVT = VT.getVectorElementType(); in expandVecReduceSeq()
8614 if (VT.isScalableVector()) in expandVecReduceSeq()
8618 unsigned NumElts = VT.getVectorNumElements(); in expandVecReduceSeq()
8634 EVT VT = Node->getValueType(0); in expandREM() local
8641 if (isOperationLegalOrCustom(DivRemOpc, VT)) { in expandREM()
8642 SDVTList VTs = DAG.getVTList(VT, VT); in expandREM()
8646 if (isOperationLegalOrCustom(DivOpc, VT)) { in expandREM()
8648 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor); in expandREM()
8649 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
8650 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
8765 EVT VT = Node->getValueType(0); in expandVectorSplice() local
8782 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false); in expandVectorSplice()
8784 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), in expandVectorSplice()
8785 VT.getVectorElementCount() * 2); in expandVectorSplice()
8797 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize())); in expandVectorSplice()
8804 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2)); in expandVectorSplice()
8806 return DAG.getLoad(VT, DL, StoreV2, StackPtr, in expandVectorSplice()
8813 TypeSize EltByteSize = VT.getVectorElementType().getStoreSize(); in expandVectorSplice()
8817 if (TrailingElts > VT.getVectorMinNumElements()) { in expandVectorSplice()
8820 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize())); in expandVectorSplice()
8828 return DAG.getLoad(VT, DL, StoreV2, StackPtr2, in expandVectorSplice()
8832 bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, in LegalizeSetCCCondCode() argument
8947 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
8948 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
8951 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
8952 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
8957 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); in LegalizeSetCCCondCode()