Lines Matching refs:Subtarget
85 : TargetLowering(TM), Subtarget(STI) { in RISCVTargetLowering()
87 RISCVABI::ABI ABI = Subtarget.getTargetABI(); in RISCVTargetLowering()
91 !Subtarget.hasStdExtF()) { in RISCVTargetLowering()
95 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32; in RISCVTargetLowering()
97 !Subtarget.hasStdExtD()) { in RISCVTargetLowering()
101 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32; in RISCVTargetLowering()
118 MVT XLenVT = Subtarget.getXLenVT(); in RISCVTargetLowering()
122 if (Subtarget.is64Bit() && RV64LegalI32) in RISCVTargetLowering()
125 if (Subtarget.hasStdExtZfhmin()) in RISCVTargetLowering()
127 if (Subtarget.hasStdExtZfbfmin()) in RISCVTargetLowering()
129 if (Subtarget.hasStdExtF()) in RISCVTargetLowering()
131 if (Subtarget.hasStdExtD()) in RISCVTargetLowering()
133 if (Subtarget.hasStdExtZhinxmin()) in RISCVTargetLowering()
135 if (Subtarget.hasStdExtZfinx()) in RISCVTargetLowering()
137 if (Subtarget.hasStdExtZdinx()) { in RISCVTargetLowering()
138 if (Subtarget.is64Bit()) in RISCVTargetLowering()
164 if (Subtarget.hasVInstructions()) { in RISCVTargetLowering()
168 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen(); in RISCVTargetLowering()
192 !Subtarget.hasVInstructionsI64()) in RISCVTargetLowering()
197 if (Subtarget.hasVInstructionsF16Minimal()) in RISCVTargetLowering()
201 if (Subtarget.hasVInstructionsBF16()) in RISCVTargetLowering()
205 if (Subtarget.hasVInstructionsF32()) in RISCVTargetLowering()
209 if (Subtarget.hasVInstructionsF64()) in RISCVTargetLowering()
213 if (Subtarget.useRVVForFixedLengthVectors()) { in RISCVTargetLowering()
217 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo(); in RISCVTargetLowering()
246 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
250 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
253 if (!Subtarget.hasVendorXCValu()) in RISCVTargetLowering()
257 if (!Subtarget.hasVendorXCValu()) in RISCVTargetLowering()
262 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
269 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
276 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb()) in RISCVTargetLowering()
279 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
288 if (!Subtarget.hasStdExtZbb()) in RISCVTargetLowering()
292 if (Subtarget.hasStdExtZbb()) { in RISCVTargetLowering()
299 if (!Subtarget.hasStdExtZmmul()) { in RISCVTargetLowering()
301 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
303 } else if (Subtarget.is64Bit()) { in RISCVTargetLowering()
313 if (!Subtarget.hasStdExtM()) { in RISCVTargetLowering()
316 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
319 } else if (Subtarget.is64Bit()) { in RISCVTargetLowering()
325 if (RV64LegalI32 && Subtarget.is64Bit()) { in RISCVTargetLowering()
339 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) { in RISCVTargetLowering()
340 if (!RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
342 } else if (Subtarget.hasVendorXTHeadBb()) { in RISCVTargetLowering()
343 if (Subtarget.is64Bit()) in RISCVTargetLowering()
346 } else if (Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
350 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
357 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() || in RISCVTargetLowering()
358 Subtarget.hasVendorXTHeadBb()) in RISCVTargetLowering()
361 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
363 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() || in RISCVTargetLowering()
364 Subtarget.hasVendorXTHeadBb()) in RISCVTargetLowering()
369 if (Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
374 Subtarget.hasStdExtZbkb() ? Custom : Expand); in RISCVTargetLowering()
377 if (Subtarget.hasStdExtZbb()) { in RISCVTargetLowering()
380 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
384 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
390 } else if (!Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
392 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
396 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || in RISCVTargetLowering()
397 Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
400 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
403 Subtarget.hasStdExtZbb() ? Legal : Promote); in RISCVTargetLowering()
404 if (!Subtarget.hasStdExtZbb()) in RISCVTargetLowering()
411 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
415 if (!RV64LegalI32 && Subtarget.is64Bit() && in RISCVTargetLowering()
416 !Subtarget.hasShortForwardBranchOpt()) in RISCVTargetLowering()
420 if (Subtarget.hasShortForwardBranchOpt()) in RISCVTargetLowering()
423 if (!Subtarget.hasVendorXTHeadCondMov()) { in RISCVTargetLowering()
425 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
450 if (Subtarget.hasStdExtZfhminOrZhinxmin()) in RISCVTargetLowering()
464 if (Subtarget.hasStdExtZfbfmin()) { in RISCVTargetLowering()
480 if (Subtarget.hasStdExtZfhminOrZhinxmin()) { in RISCVTargetLowering()
481 if (Subtarget.hasStdExtZfhOrZhinx()) { in RISCVTargetLowering()
484 Subtarget.hasStdExtZfa() ? Legal : Custom); in RISCVTargetLowering()
504 Subtarget.hasStdExtZfa() ? Legal : Promote); in RISCVTargetLowering()
520 if (Subtarget.is64Bit()) in RISCVTargetLowering()
524 Subtarget.hasStdExtZfa() ? Legal : Custom); in RISCVTargetLowering()
527 if (Subtarget.hasStdExtFOrZfinx()) { in RISCVTargetLowering()
530 Subtarget.hasStdExtZfa() ? Legal : Custom); in RISCVTargetLowering()
543 Subtarget.isSoftFPABI() ? LibCall : Custom); in RISCVTargetLowering()
547 if (Subtarget.hasStdExtZfa()) { in RISCVTargetLowering()
555 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit()) in RISCVTargetLowering()
558 if (Subtarget.hasStdExtDOrZdinx()) { in RISCVTargetLowering()
561 if (!Subtarget.is64Bit()) in RISCVTargetLowering()
564 if (Subtarget.hasStdExtZfa()) { in RISCVTargetLowering()
569 if (Subtarget.is64Bit()) in RISCVTargetLowering()
591 Subtarget.isSoftFPABI() ? LibCall : Custom); in RISCVTargetLowering()
596 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
603 if (Subtarget.hasStdExtFOrZfinx()) { in RISCVTargetLowering()
611 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
626 if (Subtarget.is64Bit()) in RISCVTargetLowering()
632 Subtarget.is64Bit() ? Legal : Custom); in RISCVTargetLowering()
634 Subtarget.is64Bit() ? Legal : Custom); in RISCVTargetLowering()
638 if (Subtarget.is64Bit()) in RISCVTargetLowering()
641 if (Subtarget.hasStdExtZicbop()) { in RISCVTargetLowering()
645 if (Subtarget.hasStdExtA()) { in RISCVTargetLowering()
646 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen()); in RISCVTargetLowering()
647 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) in RISCVTargetLowering()
651 } else if (Subtarget.hasForcedAtomics()) { in RISCVTargetLowering()
652 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen()); in RISCVTargetLowering()
666 if (Subtarget.hasVInstructions()) { in RISCVTargetLowering()
670 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
678 if (Subtarget.is64Bit()) in RISCVTargetLowering()
730 if (!Subtarget.is64Bit()) { in RISCVTargetLowering()
826 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) in RISCVTargetLowering()
899 if (Subtarget.hasStdExtZvkb()) { in RISCVTargetLowering()
907 if (Subtarget.hasStdExtZvbb()) { in RISCVTargetLowering()
1053 if (Subtarget.hasVInstructionsF16()) { in RISCVTargetLowering()
1059 } else if (Subtarget.hasVInstructionsF16Minimal()) { in RISCVTargetLowering()
1076 if (Subtarget.hasStdExtZfhmin()) in RISCVTargetLowering()
1095 if (Subtarget.hasVInstructionsBF16()) { in RISCVTargetLowering()
1107 if (Subtarget.hasStdExtZfbfmin()) in RISCVTargetLowering()
1116 if (Subtarget.hasVInstructionsF32()) { in RISCVTargetLowering()
1125 if (Subtarget.hasVInstructionsF64()) { in RISCVTargetLowering()
1135 if (Subtarget.useRVVForFixedLengthVectors()) { in RISCVTargetLowering()
1220 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) { in RISCVTargetLowering()
1245 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) in RISCVTargetLowering()
1267 if (Subtarget.hasStdExtZvkb()) in RISCVTargetLowering()
1270 if (Subtarget.hasStdExtZvbb()) { in RISCVTargetLowering()
1316 !Subtarget.hasVInstructionsF16()) { in RISCVTargetLowering()
1325 if (Subtarget.hasStdExtZfhmin()) { in RISCVTargetLowering()
1400 if (Subtarget.is64Bit()) in RISCVTargetLowering()
1402 if (Subtarget.hasStdExtZfhminOrZhinxmin()) in RISCVTargetLowering()
1404 if (Subtarget.hasStdExtFOrZfinx()) in RISCVTargetLowering()
1406 if (Subtarget.hasStdExtDOrZdinx()) in RISCVTargetLowering()
1411 if (Subtarget.hasStdExtA()) { in RISCVTargetLowering()
1413 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
1417 if (Subtarget.hasForcedAtomics()) { in RISCVTargetLowering()
1427 if (Subtarget.hasVendorXTHeadMemIdx()) { in RISCVTargetLowering()
1436 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
1443 if (Subtarget.hasVendorXCVmem()) { in RISCVTargetLowering()
1453 if (Subtarget.hasVendorXCValu()) { in RISCVTargetLowering()
1464 const Align FunctionAlignment(Subtarget.hasStdExtCOrZca() ? 2 : 4); in RISCVTargetLowering()
1467 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment()); in RISCVTargetLowering()
1468 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment()); in RISCVTargetLowering()
1473 if (Subtarget.is64Bit()) in RISCVTargetLowering()
1476 if (Subtarget.hasStdExtFOrZfinx()) in RISCVTargetLowering()
1479 if (Subtarget.hasStdExtZbb()) in RISCVTargetLowering()
1482 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) || in RISCVTargetLowering()
1483 Subtarget.hasStdExtV()) in RISCVTargetLowering()
1486 if (Subtarget.hasStdExtZbkb()) in RISCVTargetLowering()
1488 if (Subtarget.hasStdExtZfhminOrZhinxmin()) in RISCVTargetLowering()
1490 if (Subtarget.hasStdExtFOrZfinx()) in RISCVTargetLowering()
1493 if (Subtarget.hasVInstructions()) in RISCVTargetLowering()
1501 if (Subtarget.hasVendorXTHeadMemPair()) in RISCVTargetLowering()
1503 if (Subtarget.useRVVForFixedLengthVectors()) in RISCVTargetLowering()
1515 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive(); in RISCVTargetLowering()
1523 if (Subtarget.hasVInstructions() && in getSetCCResultType()
1524 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors())) in getSetCCResultType()
1530 return Subtarget.getXLenVT(); in getVPExplicitVectorLengthTy()
1537 if (!Subtarget.hasVInstructions()) in shouldExpandGetVectorLength()
1543 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT()) in shouldExpandGetVectorLength()
1547 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen()) in shouldExpandGetVectorLength()
1551 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock) in shouldExpandGetVectorLength()
1561 return !Subtarget.hasVInstructions() || in shouldExpandCttzElements()
1846 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty)) in isLegalAddressingMode()
1881 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy()) in isTruncateFree()
1902 if (Subtarget.hasStdExtV() && in isTruncateFree()
1930 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64; in isSExtCheaperThanZExt()
1934 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32); in signExtendConstant()
1938 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip(); in isCheapToSpeculateCttz()
1942 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || in isCheapToSpeculateCtlz()
1943 Subtarget.hasVendorXCVbitmanip(); in isCheapToSpeculateCtlz()
1954 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs()) in isMaskAndCmp0FoldingBeneficial()
1969 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) && in hasAndNotCompare()
1975 if (Subtarget.hasStdExtZbs()) in hasBitTest()
1979 if (Subtarget.hasVendorXTHeadBs()) in hasBitTest()
1988 if (!VT.isVector() || !Subtarget.hasVInstructions()) in shouldFoldSelectWithIdentityConstant()
2002 if (BitSize > Subtarget.getXLen()) in shouldConvertConstantLoadToIntImm()
2014 if (!Subtarget.enableUnalignedScalarMem()) in shouldConvertConstantLoadToIntImm()
2023 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget); in shouldConvertConstantLoadToIntImm()
2024 return Seq.size() <= Subtarget.getMaxBuildIntsCost(); in shouldConvertConstantLoadToIntImm()
2082 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions()) in canSplatOperand()
2147 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions()) in shouldSinkOperands()
2155 if (!Subtarget.sinkSplatOperands()) in shouldSinkOperands()
2230 if (!Subtarget.hasStdExtZfa()) in getLegalZfaFPImm()
2235 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh(); in getLegalZfaFPImm()
2239 assert(Subtarget.hasStdExtD() && "Expect D extension"); in getLegalZfaFPImm()
2258 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin(); in isFPImmLegal()
2260 IsLegalVT = Subtarget.hasStdExtFOrZfinx(); in isFPImmLegal()
2262 IsLegalVT = Subtarget.hasStdExtDOrZdinx(); in isFPImmLegal()
2264 IsLegalVT = Subtarget.hasStdExtZfbfmin(); in isFPImmLegal()
2273 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) { in isFPImmLegal()
2286 1 + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(), Subtarget.getXLen(), in isFPImmLegal()
2287 Subtarget); in isFPImmLegal()
2312 unsigned MinVLen = Subtarget.getRealMinVLen(); in isExtractSubvectorCheap()
2344 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() && in getRegisterTypeForCallingConv()
2345 !Subtarget.hasStdExtZfhminOrZhinxmin()) in getRegisterTypeForCallingConv()
2350 if (RV64LegalI32 && Subtarget.is64Bit() && PartVT == MVT::i32) in getRegisterTypeForCallingConv()
2361 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() && in getNumRegistersForCallingConv()
2362 !Subtarget.hasStdExtZfhminOrZhinxmin()) in getNumRegistersForCallingConv()
2374 if (RV64LegalI32 && Subtarget.is64Bit() && IntermediateVT == MVT::i32) in getVectorTypeBreakdownForCallingConv()
2377 if (RV64LegalI32 && Subtarget.is64Bit() && RegisterVT == MVT::i32) in getVectorTypeBreakdownForCallingConv()
2561 return !Subtarget.useRVVForFixedLengthVectors() || in mergeStoresAfterLegalization()
2570 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true; in isLegalElementTypeForRVV()
2576 return Subtarget.hasVInstructionsI64(); in isLegalElementTypeForRVV()
2578 return Subtarget.hasVInstructionsF16(); in isLegalElementTypeForRVV()
2580 return Subtarget.hasVInstructionsF32(); in isLegalElementTypeForRVV()
2582 return Subtarget.hasVInstructionsF64(); in isLegalElementTypeForRVV()
2607 const RISCVSubtarget &Subtarget) { in useRVVForFixedLengthVectorVT() argument
2609 if (!Subtarget.useRVVForFixedLengthVectors()) in useRVVForFixedLengthVectorVT()
2619 unsigned MinVLen = Subtarget.getRealMinVLen(); in useRVVForFixedLengthVectorVT()
2639 if (!Subtarget.hasVInstructionsI64()) in useRVVForFixedLengthVectorVT()
2643 if (!Subtarget.hasVInstructionsF16Minimal()) in useRVVForFixedLengthVectorVT()
2647 if (!Subtarget.hasVInstructionsBF16()) in useRVVForFixedLengthVectorVT()
2651 if (!Subtarget.hasVInstructionsF32()) in useRVVForFixedLengthVectorVT()
2655 if (!Subtarget.hasVInstructionsF64()) in useRVVForFixedLengthVectorVT()
2661 if (EltVT.getSizeInBits() > Subtarget.getELen()) in useRVVForFixedLengthVectorVT()
2666 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors()) in useRVVForFixedLengthVectorVT()
2678 return ::useRVVForFixedLengthVectorVT(VT, Subtarget); in useRVVForFixedLengthVectorVT()
2683 const RISCVSubtarget &Subtarget) { in getContainerForFixedLengthVector() argument
2686 useRVVForFixedLengthVectorVT(VT, Subtarget)) && in getContainerForFixedLengthVector()
2689 unsigned MinVLen = Subtarget.getRealMinVLen(); in getContainerForFixedLengthVector()
2690 unsigned MaxELen = Subtarget.getELen(); in getContainerForFixedLengthVector()
2718 const RISCVSubtarget &Subtarget) { in getContainerForFixedLengthVector() argument
2720 Subtarget); in getContainerForFixedLengthVector()
2729 const RISCVSubtarget &Subtarget) { in convertToScalableVector() argument
2741 const RISCVSubtarget &Subtarget) { in convertFromScalableVector() argument
2747 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in convertFromScalableVector()
2769 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { in getVLOp() argument
2774 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in getVLOp()
2776 return DAG.getRegister(RISCV::X0, Subtarget.getXLenVT()); in getVLOp()
2778 return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT()); in getVLOp()
2783 const RISCVSubtarget &Subtarget) { in getDefaultScalableVLOps() argument
2785 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT()); in getDefaultScalableVLOps()
2792 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { in getDefaultVLOps() argument
2794 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2805 const RISCVSubtarget &Subtarget) { in getDefaultVLOps() argument
2808 Subtarget); in getDefaultVLOps()
2810 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2816 return DAG.getElementCount(DL, Subtarget.getXLenVT(), in computeVLMax()
2822 const RISCVSubtarget &Subtarget) { in computeVLMAXBounds() argument
2828 unsigned VectorBitsMax = Subtarget.getRealMaxVLen(); in computeVLMAXBounds()
2832 unsigned VectorBitsMin = Subtarget.getRealMinVLen(); in computeVLMAXBounds()
2857 unsigned DLenFactor = Subtarget.getDLenFactor(); in getLMULCost()
2869 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor); in getLMULCost()
2906 const RISCVSubtarget &Subtarget) { in lowerFP_TO_INT_SAT() argument
2920 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) || in lowerFP_TO_INT_SAT()
2937 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT())); in lowerFP_TO_INT_SAT()
2966 DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget); in lowerFP_TO_INT_SAT()
2967 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in lowerFP_TO_INT_SAT()
2971 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in lowerFP_TO_INT_SAT()
2976 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget); in lowerFP_TO_INT_SAT()
2996 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL); in lowerFP_TO_INT_SAT()
3001 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget); in lowerFP_TO_INT_SAT()
3041 const RISCVSubtarget &Subtarget) { in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND() argument
3051 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
3052 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
3060 Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
3063 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
3094 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
3142 return convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
3150 const RISCVSubtarget &Subtarget) { in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND() argument
3158 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3159 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3162 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3202 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3245 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3251 const RISCVSubtarget &Subtarget) { in lowerFTRUNC_FCEIL_FFLOOR_FROUND() argument
3254 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3274 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT())); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3279 const RISCVSubtarget &Subtarget) { in lowerVectorXRINT() argument
3288 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorXRINT()
3289 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorXRINT()
3292 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorXRINT()
3299 return convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorXRINT()
3303 getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, in getVSlidedown() argument
3309 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT()); in getVSlidedown()
3315 getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, in getVSlideup() argument
3321 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT()); in getVSlideup()
3473 const RISCVSubtarget &Subtarget) { in matchSplatAsGather() argument
3484 if (Idx.getValueType() != Subtarget.getXLenVT()) in matchSplatAsGather()
3489 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in matchSplatAsGather()
3490 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in matchSplatAsGather()
3493 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in matchSplatAsGather()
3501 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in matchSplatAsGather()
3514 const RISCVSubtarget &Subtarget) { in lowerBuildVectorViaDominantValues() argument
3518 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorViaDominantValues()
3521 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3523 MVT XLenVT = Subtarget.getXLenVT(); in lowerBuildVectorViaDominantValues()
3581 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3588 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3621 const RISCVSubtarget &Subtarget) { in lowerBuildVectorOfConstants() argument
3625 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorOfConstants()
3628 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorOfConstants()
3630 MVT XLenVT = Subtarget.getXLenVT(); in lowerBuildVectorOfConstants()
3636 return convertFromScalableVector(VT, VMClr, DAG, Subtarget); in lowerBuildVectorOfConstants()
3641 return convertFromScalableVector(VT, VMSet, DAG, Subtarget); in lowerBuildVectorOfConstants()
3652 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen()); in lowerBuildVectorOfConstants()
3653 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen()); in lowerBuildVectorOfConstants()
3717 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerBuildVectorOfConstants()
3752 getContainerForFixedLengthVector(DAG, VIDVT, Subtarget); in lowerBuildVectorOfConstants()
3758 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget); in lowerBuildVectorOfConstants()
3794 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1; in lowerBuildVectorOfConstants()
3809 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32) in lowerBuildVectorOfConstants()
3833 if (VT.isInteger() && EltBitSize < Subtarget.getELen() && in lowerBuildVectorOfConstants()
3836 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) { in lowerBuildVectorOfConstants()
3848 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ? in lowerBuildVectorOfConstants()
3866 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32) in lowerBuildVectorOfConstants()
3873 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) && in lowerBuildVectorOfConstants()
3879 getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget); in lowerBuildVectorOfConstants()
3884 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget); in lowerBuildVectorOfConstants()
3900 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen())) { in lowerBuildVectorOfConstants()
3906 Source, DAG, Subtarget); in lowerBuildVectorOfConstants()
3908 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerBuildVectorOfConstants()
3912 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget)) in lowerBuildVectorOfConstants()
3922 const RISCVSubtarget &Subtarget) { in getPACKOpcode() argument
3929 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK; in getPACKOpcode()
3931 assert(Subtarget.is64Bit()); in getPACKOpcode()
3942 const RISCVSubtarget &Subtarget) { in lowerBuildVectorViaPacking() argument
3952 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba()) in lowerBuildVectorViaPacking()
3957 if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) || in lowerBuildVectorViaPacking()
3963 MVT XLenVT = Subtarget.getXLenVT(); in lowerBuildVectorViaPacking()
3971 if (Subtarget.hasStdExtZbkb()) in lowerBuildVectorViaPacking()
3975 DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget), in lowerBuildVectorViaPacking()
4011 const RISCVSubtarget &Subtarget) { in lowerBUILD_VECTOR() argument
4017 !Subtarget.hasStdExtZfhmin()) in lowerBUILD_VECTOR()
4022 return lowerBuildVectorOfConstants(Op, DAG, Subtarget); in lowerBUILD_VECTOR()
4024 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR()
4027 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBUILD_VECTOR()
4029 MVT XLenVT = Subtarget.getXLenVT(); in lowerBUILD_VECTOR()
4056 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget)) in lowerBUILD_VECTOR()
4064 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerBUILD_VECTOR()
4067 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget)) in lowerBUILD_VECTOR()
4072 if (const auto VLen = Subtarget.getRealVLen(); in lowerBUILD_VECTOR()
4076 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR()
4078 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget); in lowerBUILD_VECTOR()
4092 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget); in lowerBUILD_VECTOR()
4097 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
4104 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget)) in lowerBUILD_VECTOR()
4194 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) && in lowerBUILD_VECTOR()
4212 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
4218 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); in lowerBUILD_VECTOR()
4219 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
4226 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V); in lowerBUILD_VECTOR()
4231 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); in lowerBUILD_VECTOR()
4232 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
4235 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
4308 const RISCVSubtarget &Subtarget) { in lowerScalarSplat() argument
4315 MVT XLenVT = Subtarget.getXLenVT(); in lowerScalarSplat()
4346 const RISCVSubtarget &Subtarget) { in lowerScalarInsert() argument
4349 const MVT XLenVT = Subtarget.getXLenVT(); in lowerScalarInsert()
4362 DAG, ExtractedContainerVT, Subtarget); in lowerScalarInsert()
4364 ExtractedVal, DAG, Subtarget); in lowerScalarInsert()
4384 VT, DL, DAG, Subtarget); in lowerScalarInsert()
4406 const RISCVSubtarget &Subtarget) { in isDeinterleaveShuffle() argument
4408 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) in isDeinterleaveShuffle()
4450 int &OddSrc, const RISCVSubtarget &Subtarget) { in isInterleaveShuffle() argument
4452 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) in isInterleaveShuffle()
4561 const RISCVSubtarget &Subtarget, in getDeinterleaveViaVNSRL() argument
4568 ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget); in getDeinterleaveViaVNSRL()
4574 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4577 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4594 DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL); in getDeinterleaveViaVNSRL()
4602 Res = convertFromScalableVector(VT, Res, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4621 const RISCVSubtarget &Subtarget, in lowerVECTOR_SHUFFLEAsVSlidedown() argument
4669 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SHUFFLEAsVSlidedown()
4671 MVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown()
4672 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown()
4674 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVECTOR_SHUFFLEAsVSlidedown()
4675 convertToScalableVector(ContainerVT, Src, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlidedown()
4679 convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlidedown()
4698 const RISCVSubtarget &Subtarget, in lowerVECTOR_SHUFFLEAsVSlideup() argument
4710 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SHUFFLEAsVSlideup()
4711 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4712 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first; in lowerVECTOR_SHUFFLEAsVSlideup()
4721 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4722 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4732 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert, in lowerVECTOR_SHUFFLEAsVSlideup()
4734 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4742 const RISCVSubtarget &Subtarget, in lowerVECTOR_SHUFFLEAsVSlide1() argument
4776 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4777 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4782 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat); in lowerVECTOR_SHUFFLEAsVSlide1()
4785 convertToScalableVector(ContainerVT, V2, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlide1()
4787 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4795 const RISCVSubtarget &Subtarget) { in getWideningInterleave() argument
4800 VecContainerVT = getContainerForFixedLengthVector(DAG, VecVT, Subtarget); in getWideningInterleave()
4801 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget); in getWideningInterleave()
4802 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget); in getWideningInterleave()
4805 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen()); in getWideningInterleave()
4816 WideContainerVT = getContainerForFixedLengthVector(DAG, WideVT, Subtarget); in getWideningInterleave()
4823 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget); in getWideningInterleave()
4833 } else if (Subtarget.hasStdExtZvbb()) { in getWideningInterleave()
4861 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT())); in getWideningInterleave()
4886 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget); in getWideningInterleave()
4895 const RISCVSubtarget &Subtarget) { in lowerBitreverseShuffle() argument
4916 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE, in lowerBitreverseShuffle()
4918 !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT)) in lowerBitreverseShuffle()
4947 const RISCVSubtarget &Subtarget, in isLegalBitRotate() argument
4962 return Subtarget.getTargetLowering()->isTypeLegal(RotateVT); in isLegalBitRotate()
4970 const RISCVSubtarget &Subtarget) { in lowerVECTOR_SHUFFLEAsRotate() argument
4976 if (!isLegalBitRotate(SVN, DAG, Subtarget, RotateVT, RotateAmt)) in lowerVECTOR_SHUFFLEAsRotate()
4998 const RISCVSubtarget &Subtarget) { in lowerShuffleViaVRegSplitting() argument
5008 const auto VLen = Subtarget.getRealVLen(); in lowerShuffleViaVRegSplitting()
5016 if (isLegalBitRotate(SVN, DAG, Subtarget, RotateVT, RotateAmt)) in lowerShuffleViaVRegSplitting()
5049 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerShuffleViaVRegSplitting()
5051 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget); in lowerShuffleViaVRegSplitting()
5058 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
5059 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
5068 SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
5070 SubVec = convertToScalableVector(M1VT, SubVec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
5075 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
5079 const RISCVSubtarget &Subtarget) { in lowerVECTOR_SHUFFLE() argument
5083 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SHUFFLE()
5091 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
5093 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
5106 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLE()
5108 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5151 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5157 if (SVT == MVT::f16 && !Subtarget.hasStdExtZfh()) { in lowerVECTOR_SHUFFLE()
5182 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5185 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5190 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5196 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
5202 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
5206 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
5211 if (Subtarget.hasStdExtZvkb()) in lowerVECTOR_SHUFFLE()
5212 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
5223 LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5227 HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5238 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV, in lowerVECTOR_SHUFFLE()
5242 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV, in lowerVECTOR_SHUFFLE()
5246 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5251 if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) { in lowerVECTOR_SHUFFLE()
5253 Subtarget, DAG); in lowerVECTOR_SHUFFLE()
5257 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
5263 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget)) { in lowerVECTOR_SHUFFLE()
5279 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5289 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
5317 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) { in lowerVECTOR_SHUFFLE()
5325 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5335 Subtarget); in lowerVECTOR_SHUFFLE()
5338 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5397 isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget); in isShuffleMaskLegal()
5416 Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5456 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5459 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5461 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5467 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5522 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPCttzElements()
5530 Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget); in lowerVPCttzElements()
5532 Subtarget); in lowerVPCttzElements()
5618 const RISCVSubtarget &Subtarget) { in lowerConstant() argument
5631 if (!Subtarget.useConstantPoolForLargeInts()) in lowerConstant()
5634 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget); in lowerConstant()
5635 if (Seq.size() <= Subtarget.getMaxBuildIntsCost()) in lowerConstant()
5650 RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc); in lowerConstant()
5651 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost()) in lowerConstant()
5658 const RISCVSubtarget &Subtarget) { in LowerATOMIC_FENCE() argument
5665 if (Subtarget.hasStdExtZtso()) { in LowerATOMIC_FENCE()
5762 MVT XLenVT = Subtarget.getXLenVT(); in LowerIS_FPCLASS()
5796 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget); in LowerIS_FPCLASS()
5816 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget); in LowerIS_FPCLASS()
5821 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget); in LowerIS_FPCLASS()
5824 Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget); in LowerIS_FPCLASS()
5836 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget); in LowerIS_FPCLASS()
5848 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget); in LowerIS_FPCLASS()
5861 const RISCVSubtarget &Subtarget) { in lowerFMAXIMUM_FMINIMUM() argument
5869 MVT XLenVT = Subtarget.getXLenVT(); in lowerFMAXIMUM_FMINIMUM()
5899 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5900 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5901 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5909 Subtarget); in lowerFMAXIMUM_FMINIMUM()
5912 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5940 Res = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
6249 return LowerATOMIC_FENCE(Op, DAG, Subtarget); in LowerOperation()
6261 return lowerConstant(Op, DAG, Subtarget); in LowerOperation()
6286 assert(Subtarget.hasStdExtZvkb()); in LowerOperation()
6289 assert(Subtarget.hasVendorXTHeadBb() && in LowerOperation()
6290 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) && in LowerOperation()
6301 MVT XLenVT = Subtarget.getXLenVT(); in LowerOperation()
6303 Subtarget.hasStdExtZfhminOrZhinxmin()) { in LowerOperation()
6309 Subtarget.hasStdExtZfbfmin()) { in LowerOperation()
6314 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() && in LowerOperation()
6315 Subtarget.hasStdExtFOrZfinx()) { in LowerOperation()
6321 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() && in LowerOperation()
6322 Subtarget.hasStdExtDOrZdinx()) { in LowerOperation()
6383 assert(Subtarget.hasStdExtZvbb()); in LowerOperation()
6387 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization"); in LowerOperation()
6427 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerOperation()
6428 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar); in LowerOperation()
6432 V = convertFromScalableVector(VT, V, DAG, Subtarget); in LowerOperation()
6436 MVT XLenVT = Subtarget.getXLenVT(); in LowerOperation()
6444 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock) in LowerOperation()
6473 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() && in LowerOperation()
6487 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6488 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6490 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget); in LowerOperation()
6496 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) in LowerOperation()
6498 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) { in LowerOperation()
6513 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin()) in LowerOperation()
6515 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() && in LowerOperation()
6516 Subtarget.hasStdExtDOrZdinx()) { in LowerOperation()
6534 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6535 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6553 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6554 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6691 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in LowerOperation()
6693 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in LowerOperation()
6697 SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget); in LowerOperation()
6701 return convertFromScalableVector(VT, Src, DAG, Subtarget); in LowerOperation()
6705 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget); in LowerOperation()
6709 assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization"); in LowerOperation()
6716 if (Subtarget.is64Bit() && !RV64LegalI32) in LowerOperation()
6721 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization"); in LowerOperation()
6727 SDValue Res = Subtarget.is64Bit() in LowerOperation()
6738 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation"); in LowerOperation()
6745 if (Subtarget.is64Bit() && !RV64LegalI32) in LowerOperation()
6752 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation"); in LowerOperation()
6755 SDValue Arg = Subtarget.is64Bit() in LowerOperation()
6771 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in LowerOperation()
6774 return lowerVectorXRINT(Op, DAG, Subtarget); in LowerOperation()
6806 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6807 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6822 DAG.getUNDEF(ContainerVT), DAG, Subtarget); in LowerOperation()
6839 return lowerBUILD_VECTOR(Op, DAG, Subtarget); in LowerOperation()
6842 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6843 Subtarget.hasStdExtZfhminOrZhinxmin() && in LowerOperation()
6844 !Subtarget.hasVInstructionsF16())) || in LowerOperation()
6846 (Subtarget.hasVInstructionsBF16() && Subtarget.hasStdExtZfbfmin()))) { in LowerOperation()
6864 return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget); in LowerOperation()
6872 ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget); in LowerOperation()
6982 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6983 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7009 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() && in LowerOperation()
7023 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7024 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7069 if (Subtarget.hasStdExtZvbb()) in LowerOperation()
7077 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7078 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7088 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7089 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7102 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in LowerOperation()
7147 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7148 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7171 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7172 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
7190 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7191 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
7206 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7207 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7221 if (Subtarget.hasStdExtZvbb()) in LowerOperation()
7226 if (Subtarget.hasStdExtZvbb()) in LowerOperation()
7243 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7244 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7246 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in LowerOperation()
7250 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
7251 !Subtarget.hasVInstructionsF16())) in LowerOperation()
7253 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget); in LowerOperation()
7264 SDValue Flags = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in LowerOperation()
7315 if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) { in getAddr()
7317 if (IsLocal && !Subtarget.allowTaggedGlobals()) in getAddr()
7411 MVT XLenVT = Subtarget.getXLenVT(); in getStaticTLSAddr()
7560 const RISCVSubtarget &Subtarget) { in combineSelectToBinOp() argument
7567 if (!Subtarget.hasConditionalMoveFusion()) { in combineSelectToBinOp()
7636 const RISCVSubtarget &Subtarget) { in foldBinOpIntoSelectIfProfitable() argument
7637 if (Subtarget.hasShortForwardBranchOpt()) in foldBinOpIntoSelectIfProfitable()
7699 MVT XLenVT = Subtarget.getXLenVT(); in lowerSELECT()
7712 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) && in lowerSELECT()
7735 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget)) in lowerSELECT()
7744 TrueVal, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true); in lowerSELECT()
7746 FalseVal, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true); in lowerSELECT()
7760 if (!Subtarget.hasConditionalMoveFusion()) in lowerSELECT()
7767 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget)) in lowerSELECT()
7775 DAG, Subtarget)) { in lowerSELECT()
7874 MVT XLenVT = Subtarget.getXLenVT(); in lowerBRCOND()
7911 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); in lowerFRAMEADDR()
7916 int XLenInBytes = Subtarget.getXLen() / 8; in lowerFRAMEADDR()
7934 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); in lowerRETURNADDR()
7938 MVT XLenVT = Subtarget.getXLenVT(); in lowerRETURNADDR()
7939 int XLenInBytes = Subtarget.getXLen() / 8; in lowerRETURNADDR()
7979 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftLeftParts()
7980 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftLeftParts()
8029 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftRightParts()
8030 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftRightParts()
8062 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
8066 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
8085 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 && in lowerSPLAT_VECTOR_PARTS()
8096 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerSPLAT_VECTOR_PARTS()
8102 Res = convertFromScalableVector(VecVT, Res, DAG, Subtarget); in lowerSPLAT_VECTOR_PARTS()
8130 SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget); in lowerVectorMaskExt()
8132 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerVectorMaskExt()
8134 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorMaskExt()
8146 return convertFromScalableVector(VecVT, Select, DAG, Subtarget); in lowerVectorMaskExt()
8166 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
8169 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
8173 return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
8199 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorMaskTruncLike()
8203 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget); in lowerVectorMaskTruncLike()
8209 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskTruncLike()
8212 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT()); in lowerVectorMaskTruncLike()
8213 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in lowerVectorMaskTruncLike()
8227 Trunc = convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget); in lowerVectorMaskTruncLike()
8265 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorTruncLike()
8268 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVectorTruncLike()
8275 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorTruncLike()
8288 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVectorTruncLike()
8306 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
8309 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
8336 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
8381 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
8384 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
8390 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
8397 Src = convertFromScalableVector(VT, Src, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
8410 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
8421 const RISCVSubtarget &Subtarget) { in getSmallestVTForIndex() argument
8424 const unsigned VectorBitsMin = Subtarget.getRealMinVLen(); in getSmallestVTForIndex()
8467 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8479 DL, DAG, Subtarget)) { in lowerINSERT_VECTOR_ELT()
8488 if (auto VLEN = Subtarget.getRealVLen(); in lowerINSERT_VECTOR_ELT()
8506 MVT XLenVT = Subtarget.getXLenVT(); in lowerINSERT_VECTOR_ELT()
8508 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64; in lowerINSERT_VECTOR_ELT()
8521 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8538 return convertFromScalableVector(VecVT, Vec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8540 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8551 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_VECTOR_ELT()
8574 return convertFromScalableVector(VecVT, ValInVec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8599 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec, in lowerINSERT_VECTOR_ELT()
8607 return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8621 MVT XLenVT = Subtarget.getXLenVT(); in lowerEXTRACT_VECTOR_ELT()
8629 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8631 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8645 unsigned MaxEEW = Subtarget.getELen(); in lowerEXTRACT_VECTOR_ELT()
8689 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8696 const auto VLen = Subtarget.getRealVLen(); in lowerEXTRACT_VECTOR_ELT()
8722 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_VECTOR_ELT()
8748 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8749 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_VECTOR_ELT()
8766 const RISCVSubtarget &Subtarget) { in lowerVectorIntrinsicScalars() argument
8772 if (!Subtarget.hasVInstructions()) in lowerVectorIntrinsicScalars()
8792 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorIntrinsicScalars()
8853 RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget); in lowerVectorIntrinsicScalars()
8954 const RISCVSubtarget &Subtarget) { in lowerGetVectorLength() argument
8955 MVT XLenVT = Subtarget.getXLenVT(); in lowerGetVectorLength()
8964 RISCV::RVVBitsPerBlock / Subtarget.getELen(); in lowerGetVectorLength()
8989 const RISCVSubtarget &Subtarget) { in lowerCttzElts() argument
8994 ContainerVT = getContainerForFixedLengthVector(DAG, OpVT, Subtarget); in lowerCttzElts()
8995 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget); in lowerCttzElts()
8997 MVT XLenVT = Subtarget.getXLenVT(); in lowerCttzElts()
8999 auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget); in lowerCttzElts()
9014 const RISCVSubtarget &Subtarget = in promoteVCIXScalar() local
9032 MVT XLenVT = Subtarget.getXLenVT(); in promoteVCIXScalar()
9053 const RISCVSubtarget &Subtarget = in processVCIXOperands() local
9065 DAG, V.getSimpleValueType(), Subtarget); in processVCIXOperands()
9066 V = convertToScalableVector(OpContainerVT, V, DAG, Subtarget); in processVCIXOperands()
9073 const RISCVSubtarget &Subtarget) { in isValidEGW() argument
9074 return (Subtarget.getRealMinVLen() * in isValidEGW()
9083 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_WO_CHAIN()
9112 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
9126 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
9146 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
9159 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
9173 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
9187 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
9205 return lowerGetVectorLength(Op.getNode(), DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
9207 return lowerCttzElts(Op.getNode(), DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
9218 Subtarget); in LowerINTRINSIC_WO_CHAIN()
9287 if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
9288 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
9289 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget)) in LowerINTRINSIC_WO_CHAIN()
9296 if (!isValidEGW(8, Op.getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
9297 !isValidEGW(8, Op->getOperand(1).getSimpleValueType(), Subtarget)) in LowerINTRINSIC_WO_CHAIN()
9306 !Subtarget.hasStdExtZvknhb()) in LowerINTRINSIC_WO_CHAIN()
9308 if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
9309 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
9310 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget)) in LowerINTRINSIC_WO_CHAIN()
9343 NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
9354 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
9363 const RISCVSubtarget &Subtarget = in getVCIXISDNodeWCHAIN() local
9376 Subtarget); in getVCIXISDNodeWCHAIN()
9385 NewNode = convertFromScalableVector(FloatVT, NewNode, DAG, Subtarget); in getVCIXISDNodeWCHAIN()
9411 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_W_CHAIN()
9427 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
9428 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
9433 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_W_CHAIN()
9447 Subtarget); in LowerINTRINSIC_W_CHAIN()
9481 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
9499 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_W_CHAIN()
9504 Subtarget); in LowerINTRINSIC_W_CHAIN()
9520 DAG, Subtarget)); in LowerINTRINSIC_W_CHAIN()
9554 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
9565 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_VOID()
9577 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in LowerINTRINSIC_VOID()
9582 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in LowerINTRINSIC_VOID()
9585 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_VOID()
9620 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_VOID()
9625 Subtarget); in LowerINTRINSIC_VOID()
9633 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget)); in LowerINTRINSIC_VOID()
9666 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget); in LowerINTRINSIC_VOID()
9725 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorMaskVecReduction()
9730 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVectorMaskVecReduction()
9739 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskVecReduction()
9805 const RISCVSubtarget &Subtarget) { in lowerReductionSeq() argument
9808 const MVT XLenVT = Subtarget.getXLenVT(); in lowerReductionSeq()
9819 DAG, Subtarget); in lowerReductionSeq()
9861 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVECREDUCE()
9864 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVECREDUCE()
9878 Mask, VL, DL, DAG, Subtarget); in lowerVECREDUCE()
9886 const RISCVSubtarget &Subtarget) { in getRVVFPReductionOpAndOperands() argument
9926 getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT, Subtarget); in lowerFPVECREDUCE()
9932 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget); in lowerFPVECREDUCE()
9936 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9938 VL, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9951 MVT XLenVT = Subtarget.getXLenVT(); in lowerFPVECREDUCE()
9968 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPREDUCE()
9980 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVPREDUCE()
9987 Vec, Mask, VL, DL, DAG, Subtarget); in lowerVPREDUCE()
10019 MVT XLenVT = Subtarget.getXLenVT(); in lowerINSERT_SUBVECTOR()
10021 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); in lowerINSERT_SUBVECTOR()
10065 const auto VLen = Subtarget.getRealVLen(); in lowerINSERT_SUBVECTOR()
10072 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10079 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10087 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_SUBVECTOR()
10091 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10105 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
10110 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10117 Vec = convertToScalableVector(ContainerVecVT, Vec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10123 SubVec = convertToScalableVector(ContainerSubVecVT, SubVec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10150 Subtarget.expandVScale(SubVecVT.getSizeInBits()).getKnownMinValue())); in lowerINSERT_SUBVECTOR()
10152 Subtarget.expandVScale(SubVecVT.getSizeInBits()) in lowerINSERT_SUBVECTOR()
10153 .isKnownMultipleOf(Subtarget.expandVScale(VecRegSize)); in lowerINSERT_SUBVECTOR()
10180 Insert = convertFromScalableVector(VecVT, Insert, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10208 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10215 if (Subtarget.expandVScale(EndIndex) == in lowerINSERT_SUBVECTOR()
10216 Subtarget.expandVScale(InterSubVT.getVectorElementCount())) in lowerINSERT_SUBVECTOR()
10230 SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec, in lowerINSERT_SUBVECTOR()
10241 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
10255 MVT XLenVT = Subtarget.getXLenVT(); in lowerEXTRACT_SUBVECTOR()
10257 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); in lowerEXTRACT_SUBVECTOR()
10301 const auto VLen = Subtarget.getRealVLen(); in lowerEXTRACT_SUBVECTOR()
10312 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
10318 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_SUBVECTOR()
10325 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerEXTRACT_SUBVECTOR()
10329 Subtarget); in lowerEXTRACT_SUBVECTOR()
10332 getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_SUBVECTOR()
10342 Vec = convertToScalableVector(VecVT, Vec, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
10377 return convertFromScalableVector(SubVecVT, Vec, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
10402 auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
10405 Subtarget); in lowerEXTRACT_SUBVECTOR()
10407 getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT), in lowerEXTRACT_SUBVECTOR()
10489 auto [Mask, VL] = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget); in lowerVECTOR_DEINTERLEAVE()
10494 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) { in lowerVECTOR_DEINTERLEAVE()
10496 getDeinterleaveViaVNSRL(DL, VecVT, Concat, true, Subtarget, DAG); in lowerVECTOR_DEINTERLEAVE()
10498 getDeinterleaveViaVNSRL(DL, VecVT, Concat, false, Subtarget, DAG); in lowerVECTOR_DEINTERLEAVE()
10538 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_INTERLEAVE()
10563 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) { in lowerVECTOR_INTERLEAVE()
10565 DAG, Subtarget); in lowerVECTOR_INTERLEAVE()
10622 MVT XLenVT = Subtarget.getXLenVT(); in lowerSTEP_VECTOR()
10623 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
10635 VL, VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
10659 unsigned VectorBitsMax = Subtarget.getRealMaxVLen(); in lowerVECTOR_REVERSE()
10695 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_REVERSE()
10696 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget); in lowerVECTOR_REVERSE()
10705 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64; in lowerVECTOR_REVERSE()
10726 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SPLICE()
10748 getVSlidedown(DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1, in lowerVECTOR_SPLICE()
10750 return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset, in lowerVECTOR_SPLICE()
10767 MVT XLenVT = Subtarget.getXLenVT(); in lowerFixedLengthVectorLoadToRVV()
10773 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10781 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10785 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10800 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10817 MVT XLenVT = Subtarget.getXLenVT(); in lowerFixedLengthVectorStoreToRVV()
10830 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget); in lowerFixedLengthVectorStoreToRVV()
10836 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in lowerFixedLengthVectorStoreToRVV()
10846 Subtarget); in lowerFixedLengthVectorStoreToRVV()
10881 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedLoad()
10886 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in lowerMaskedLoad()
10889 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedLoad()
10894 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedLoad()
10917 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerMaskedLoad()
10949 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedStore()
10955 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in lowerMaskedStore()
10958 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedStore()
10963 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedStore()
10996 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10998 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
11002 DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
11009 return convertFromScalableVector(VT, Cmp, DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
11057 Op1 = convertToScalableVector(ContainerInVT, Op1, DAG, Subtarget); in lowerVectorStrictFSetcc()
11058 Op2 = convertToScalableVector(ContainerInVT, Op2, DAG, Subtarget); in lowerVectorStrictFSetcc()
11062 auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget); in lowerVectorStrictFSetcc()
11094 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVectorStrictFSetcc()
11112 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget); in lowerABS()
11120 Subtarget); in lowerABS()
11123 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerABS()
11127 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL); in lowerABS()
11134 Max = convertFromScalableVector(VT, Max, DAG, Subtarget); in lowerABS()
11148 Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
11149 Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
11151 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
11156 return convertFromScalableVector(VT, CopySign, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
11168 convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
11170 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
11172 convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
11175 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerFixedLengthVectorSelectToRVV()
11180 return convertFromScalableVector(VT, Select, DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
11206 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); in lowerToScalableOp()
11210 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerToScalableOp()
11223 SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget); in lowerToScalableOp()
11229 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget); in lowerToScalableOp()
11281 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); in lowerVPOp()
11289 return convertFromScalableVector(VT, VPOp, DAG, Subtarget); in lowerVPOp()
11305 Src = convertToScalableVector(SrcVT, Src, DAG, Subtarget); in lowerVPExtMaskOp()
11308 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPExtMaskOp()
11322 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPExtMaskOp()
11339 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPSetCCMaskOp()
11340 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerVPSetCCMaskOp()
11401 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSetCCMaskOp()
11419 Src = convertToScalableVector(SrcVT, Src, DAG, Subtarget); in lowerVPFPIntConvOp()
11421 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPFPIntConvOp()
11439 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPFPIntConvOp()
11507 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPFPIntConvOp()
11535 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPFPIntConvOp()
11550 const MVT XLenVT = Subtarget.getXLenVT(); in lowerVPSpliceExperimental()
11555 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPSpliceExperimental()
11556 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerVPSpliceExperimental()
11558 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPSpliceExperimental()
11603 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
11605 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2, in lowerVPSpliceExperimental()
11619 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSpliceExperimental()
11634 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPSplatExperimental()
11638 lowerScalarSplat(SDValue(), Val, VL, ContainerVT, DL, DAG, Subtarget); in lowerVPSplatExperimental()
11642 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSplatExperimental()
11650 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPReverseExperimental()
11659 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPReverseExperimental()
11661 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPReverseExperimental()
11684 unsigned VectorBitsMax = Subtarget.getRealMaxVLen(); in lowerVPReverseExperimental()
11721 Result = getVSlidedown(DAG, Subtarget, DL, GatherVT, in lowerVPReverseExperimental()
11735 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPReverseExperimental()
11763 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPReverseExperimental()
11781 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerLogicVPOp()
11782 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerLogicVPOp()
11789 return convertFromScalableVector(VT, Val, DAG, Subtarget); in lowerLogicVPOp()
11795 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPStridedLoad()
11817 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPStridedLoad()
11833 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPStridedLoad()
11841 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPStridedStore()
11849 StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget); in lowerVPStridedStore()
11864 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPStridedStore()
11912 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedGather()
11931 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget); in lowerMaskedGather()
11935 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedGather()
11936 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in lowerMaskedGather()
11941 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedGather()
11969 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerMaskedGather()
12010 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedScatter()
12029 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget); in lowerMaskedScatter()
12030 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in lowerMaskedScatter()
12034 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedScatter()
12039 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedScatter()
12062 const MVT XLenVT = Subtarget.getXLenVT(); in lowerGET_ROUNDING()
12093 const MVT XLenVT = Subtarget.getXLenVT(); in lowerSET_ROUNDING()
12127 bool isRISCV64 = Subtarget.is64Bit(); in lowerEH_DWARF_CFA()
12198 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12212 !Subtarget.hasStdExtZfhOrZhinx()) { in ReplaceNodeResults()
12230 !Subtarget.hasStdExtZfhOrZhinx()) || in ReplaceNodeResults()
12271 if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) in ReplaceNodeResults()
12295 assert(!Subtarget.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only " in ReplaceNodeResults()
12299 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
12338 unsigned XLen = Subtarget.getXLen(); in ReplaceNodeResults()
12353 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
12377 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12384 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12388 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() && in ReplaceNodeResults()
12413 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12415 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() || in ReplaceNodeResults()
12416 Subtarget.hasVendorXTHeadBb()) && in ReplaceNodeResults()
12419 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb())) in ReplaceNodeResults()
12427 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12444 Subtarget.is64Bit() && Subtarget.hasStdExtM() && in ReplaceNodeResults()
12465 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12499 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12538 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12540 if (Subtarget.hasStdExtZbb()) { in ReplaceNodeResults()
12560 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12566 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12569 if (Subtarget.hasStdExtZbb()) { in ReplaceNodeResults()
12607 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
12609 Subtarget.hasStdExtZfhminOrZhinxmin()) { in ReplaceNodeResults()
12613 Subtarget.hasStdExtZfbfmin()) { in ReplaceNodeResults()
12616 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() && in ReplaceNodeResults()
12617 Subtarget.hasStdExtFOrZfinx()) { in ReplaceNodeResults()
12621 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() && in ReplaceNodeResults()
12622 Subtarget.hasStdExtDOrZdinx()) { in ReplaceNodeResults()
12646 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
12647 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) && in ReplaceNodeResults()
12649 assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) || in ReplaceNodeResults()
12650 (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) && in ReplaceNodeResults()
12677 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 && in ReplaceNodeResults()
12685 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in ReplaceNodeResults()
12688 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
12691 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in ReplaceNodeResults()
12696 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in ReplaceNodeResults()
12724 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget); in ReplaceNodeResults()
12729 SDValue Res = lowerCttzElts(N, DAG, Subtarget); in ReplaceNodeResults()
12742 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
12776 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
12787 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
12800 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
12813 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
12843 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
12847 Subtarget.getXLenVT(), N->getOperand(1)); in ReplaceNodeResults()
12852 assert(VT == MVT::i64 && !Subtarget.is64Bit() && in ReplaceNodeResults()
12864 auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget); in ReplaceNodeResults()
12903 SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other); in ReplaceNodeResults()
12948 const RISCVSubtarget &Subtarget) { in combineBinOpOfExtractToReduceTree() argument
12958 if (!Subtarget.hasVInstructions()) in combineBinOpOfExtractToReduceTree()
12996 if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen()) in combineBinOpOfExtractToReduceTree()
13044 const RISCVSubtarget &Subtarget) { in combineBinOpToReduce() argument
13130 ScalarV.getSimpleValueType(), DL, DAG, Subtarget); in combineBinOpToReduce()
13150 const RISCVSubtarget &Subtarget) { in transformAddShlImm() argument
13152 if (!Subtarget.hasStdExtZba()) in transformAddShlImm()
13157 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen()) in transformAddShlImm()
13206 const RISCVSubtarget &Subtarget) { in combineSelectAndUse() argument
13213 if (!Subtarget.hasConditionalMoveFusion()) { in combineSelectAndUse()
13215 if ((!Subtarget.hasStdExtZicond() && in combineSelectAndUse()
13216 !Subtarget.hasVendorXVentanaCondOps()) || in combineSelectAndUse()
13225 if (VT.getSizeInBits() > Subtarget.getXLen()) in combineSelectAndUse()
13271 const RISCVSubtarget &Subtarget) { in combineSelectAndUseCommutative() argument
13274 if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes, Subtarget)) in combineSelectAndUseCommutative()
13276 if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes, Subtarget)) in combineSelectAndUseCommutative()
13300 const RISCVSubtarget &Subtarget) { in transformAddImmMulImm() argument
13303 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen()) in transformAddImmMulImm()
13427 const RISCVSubtarget &Subtarget) { in performADDCombine() argument
13431 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget)) in performADDCombine()
13434 if (SDValue V = transformAddShlImm(N, DAG, Subtarget)) in performADDCombine()
13436 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performADDCombine()
13438 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performADDCombine()
13445 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget); in performADDCombine()
13492 const RISCVSubtarget &Subtarget) { in combineSubShiftToOrcB() argument
13493 if (!Subtarget.hasStdExtZbb()) in combineSubShiftToOrcB()
13498 if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16) in combineSubShiftToOrcB()
13519 const RISCVSubtarget &Subtarget) { in performSUBCombine() argument
13540 if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget)) in performSUBCombine()
13545 return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false, Subtarget); in performSUBCombine()
13666 const RISCVSubtarget &Subtarget) { in performTRUNCATECombine() argument
13675 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 && in performTRUNCATECombine()
13693 const RISCVSubtarget &Subtarget) { in performANDCombine() argument
13702 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && in performANDCombine()
13715 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performANDCombine()
13717 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performANDCombine()
13726 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget); in performANDCombine()
13766 const RISCVSubtarget &Subtarget) { in performORCombine() argument
13769 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performORCombine()
13771 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performORCombine()
13789 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget); in performORCombine()
13793 const RISCVSubtarget &Subtarget) { in performXORCombine() argument
13800 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && in performXORCombine()
13854 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performXORCombine()
13856 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performXORCombine()
13861 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget); in performXORCombine()
13867 const RISCVSubtarget &Subtarget) { in expandMul() argument
13878 if (VT != Subtarget.getXLenVT()) in expandMul()
13882 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa(); in expandMul()
14056 const RISCVSubtarget &Subtarget) { in performMULCombine() argument
14059 return expandMul(N, DAG, DCI, Subtarget); in performMULCombine()
14175 const RISCVSubtarget &Subtarget) { in performSETCCCombine() argument
14181 if (OpVT != MVT::i64 || !Subtarget.is64Bit()) in performSETCCCombine()
14222 const RISCVSubtarget &Subtarget) { in performSIGN_EXTEND_INREGCombine() argument
14321 const RISCVSubtarget &Subtarget, in getOrCreateExtendedOp()
14329 assert(Subtarget.getTargetLowering()->isTypeLegal(Source.getValueType())); in getOrCreateExtendedOp()
14337 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget); in getOrCreateExtendedOp()
14490 const RISCVSubtarget &Subtarget) { in fillUpExtensionSupportForSplat()
14513 !Subtarget.is64Bit() && "Unexpected splat"); in fillUpExtensionSupportForSplat()
14544 const RISCVSubtarget &Subtarget) { in fillUpExtensionSupport()
14581 fillUpExtensionSupportForSplat(Root, DAG, Subtarget); in fillUpExtensionSupport()
14608 const RISCVSubtarget &Subtarget) { in isSupportedRoot()
14636 Subtarget.hasStdExtZvbb(); in isSupportedRoot()
14638 return Subtarget.hasStdExtZvbb(); in isSupportedRoot()
14646 const RISCVSubtarget &Subtarget) { in NodeExtensionHelper()
14647 assert(isSupportedRoot(Root, Subtarget) && in NodeExtensionHelper()
14680 fillUpExtensionSupport(Root, DAG, Subtarget); in NodeExtensionHelper()
14688 const RISCVSubtarget &Subtarget) { in getMaskAndVL()
14689 assert(isSupportedRoot(Root, Subtarget) && "Unexpected root"); in getMaskAndVL()
14698 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in getMaskAndVL()
14769 const RISCVSubtarget &Subtarget) const { in materialize()
14772 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget); in materialize()
14786 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt), in materialize()
14787 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt), in materialize()
14806 const RISCVSubtarget &Subtarget) { in canFoldToVWWithSameExtensionImpl() argument
14831 const RISCVSubtarget &Subtarget) { in canFoldToVWWithSameExtension() argument
14834 Subtarget); in canFoldToVWWithSameExtension()
14844 const RISCVSubtarget &Subtarget) { in canFoldToVW_W() argument
14872 const RISCVSubtarget &Subtarget) { in canFoldToVWWithSEXT() argument
14874 Subtarget); in canFoldToVWWithSEXT()
14884 const RISCVSubtarget &Subtarget) { in canFoldToVWWithZEXT() argument
14886 Subtarget); in canFoldToVWWithZEXT()
14896 const RISCVSubtarget &Subtarget) { in canFoldToVWWithFPEXT() argument
14898 Subtarget); in canFoldToVWWithFPEXT()
14908 const RISCVSubtarget &Subtarget) { in canFoldToVW_SU() argument
14985 const RISCVSubtarget &Subtarget) { in combineBinOp_VLToVWBinOp_VL() argument
14990 if (!NodeExtensionHelper::isSupportedRoot(N, Subtarget)) in combineBinOp_VLToVWBinOp_VL()
15001 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget)) in combineBinOp_VLToVWBinOp_VL()
15004 NodeExtensionHelper LHS(Root, 0, DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
15005 NodeExtensionHelper RHS(Root, 1, DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
15033 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
15062 SDValue NewValue = Res.materialize(DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
15125 const RISCVSubtarget &Subtarget) { in performVWADDSUBW_VLCombine() argument
15130 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in performVWADDSUBW_VLCombine()
15150 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>(); in tryMemPairCombine() local
15153 MVT XLenVT = Subtarget.getXLenVT(); in tryMemPairCombine()
15201 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>(); in performMemPairCombine() local
15204 if (!Subtarget.hasVendorXTHeadMemPair()) in performMemPairCombine()
15286 const RISCVSubtarget &Subtarget) { in performFP_TO_INTCombine() argument
15289 MVT XLenVT = Subtarget.getXLenVT(); in performFP_TO_INTCombine()
15302 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh()) in performFP_TO_INTCombine()
15330 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in performFP_TO_INTCombine()
15331 XVal = convertToScalableVector(SrcContainerVT, XVal, DAG, Subtarget); in performFP_TO_INTCombine()
15333 getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget); in performFP_TO_INTCombine()
15337 getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget); in performFP_TO_INTCombine()
15359 FpToInt = convertFromScalableVector(VT, FpToInt, DAG, Subtarget); in performFP_TO_INTCombine()
15389 const RISCVSubtarget &Subtarget) { in performFP_TO_INT_SATCombine() argument
15392 MVT XLenVT = Subtarget.getXLenVT(); in performFP_TO_INT_SATCombine()
15411 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh()) in performFP_TO_INT_SATCombine()
15451 const RISCVSubtarget &Subtarget) { in performBITREVERSECombine() argument
15452 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension"); in performBITREVERSECombine()
15459 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() || in performBITREVERSECombine()
15546 const RISCVSubtarget &Subtarget) { in performVFMADD_VLCombine() argument
15551 !Subtarget.hasVInstructionsF16()) in performVFMADD_VLCombine()
15605 const RISCVSubtarget &Subtarget) { in performSRACombine() argument
15608 if (N->getValueType(0) != MVT::i64 || !Subtarget.is64Bit()) in performSRACombine()
15776 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { in combine_CC() argument
15795 LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) { in combine_CC()
15978 const RISCVSubtarget &Subtarget) { in useInversedSetcc() argument
15991 if (Subtarget.hasStdExtZbs() && VT.isScalarInteger() && in useInversedSetcc()
15992 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) { in useInversedSetcc()
16009 const RISCVSubtarget &Subtarget) { in performSELECTCombine() argument
16013 if (SDValue V = useInversedSetcc(N, DAG, Subtarget)) in performSELECTCombine()
16016 if (Subtarget.hasConditionalMoveFusion()) in performSELECTCombine()
16032 const RISCVSubtarget &Subtarget, in performBUILD_VECTORCombine() argument
16090 const RISCVSubtarget &Subtarget, in performINSERT_VECTOR_ELTCombine() argument
16162 const RISCVSubtarget &Subtarget, in performCONCAT_VECTORSCombine() argument
16278 DAG.getConstant(N->getNumOperands(), DL, Subtarget.getXLenVT()), MMO); in performCONCAT_VECTORSCombine()
16287 const RISCVSubtarget &Subtarget) { in combineToVWMACC() argument
16326 const RISCVSubtarget &Subtarget) { in combineToVWMACC() argument
16330 Subtarget); in combineToVWMACC()
16333 }(N, DAG, Subtarget); in combineToVWMACC()
16528 const RISCVSubtarget &Subtarget) { in combineTruncToVnclip() argument
16650 DAG.getTargetConstant(RISCVVXRndMode::RNU, DL, Subtarget.getXLenVT()), in combineTruncToVnclip()
16660 const MVT XLenVT = Subtarget.getXLenVT(); in PerformDAGCombine()
16803 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
16805 if (SDValue V = combineToVWMACC(N, DAG, Subtarget)) in PerformDAGCombine()
16807 return performADDCombine(N, DCI, Subtarget); in PerformDAGCombine()
16810 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
16812 return performSUBCombine(N, DAG, Subtarget); in PerformDAGCombine()
16815 return performANDCombine(N, DCI, Subtarget); in PerformDAGCombine()
16817 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
16819 return performORCombine(N, DCI, Subtarget); in PerformDAGCombine()
16822 return performXORCombine(N, DAG, Subtarget); in PerformDAGCombine()
16824 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
16826 return performMULCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
16841 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in PerformDAGCombine()
16843 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in PerformDAGCombine()
16848 return performSETCCCombine(N, DAG, Subtarget); in PerformDAGCombine()
16850 return performSIGN_EXTEND_INREGCombine(N, DAG, Subtarget); in PerformDAGCombine()
16855 if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) { in PerformDAGCombine()
16876 return combineTruncToVnclip(N, DAG, Subtarget); in PerformDAGCombine()
16878 return performTRUNCATECombine(N, DAG, Subtarget); in PerformDAGCombine()
16880 return performSELECTCombine(N, DAG, Subtarget); in PerformDAGCombine()
16932 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) && in PerformDAGCombine()
16946 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT)); in PerformDAGCombine()
16957 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget)) in PerformDAGCombine()
16961 if (!Subtarget.hasConditionalMoveFusion()) { in PerformDAGCombine()
17024 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget)) in PerformDAGCombine()
17031 return performBITREVERSECombine(N, DAG, Subtarget); in PerformDAGCombine()
17034 return performFP_TO_INTCombine(N, DCI, Subtarget); in PerformDAGCombine()
17037 return performFP_TO_INT_SATCombine(N, DCI, Subtarget); in PerformDAGCombine()
17091 isSimpleVIDSequence(Index, Subtarget.getXLen()); in PerformDAGCombine()
17106 SDValue EVL = DAG.getElementCount(DL, Subtarget.getXLenVT(), in PerformDAGCombine()
17134 MGN->getMemOperand()->getBaseAlign(), Subtarget)) { in PerformDAGCombine()
17249 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
17268 if (SDValue V = performSRACombine(N, DAG, Subtarget)) in PerformDAGCombine()
17274 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
17284 DAG.getRegister(RISCV::X0, Subtarget.getXLenVT())); in PerformDAGCombine()
17290 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
17292 return combineToVWMACC(N, DAG, Subtarget); in PerformDAGCombine()
17297 return performVWADDSUBW_VLCombine(N, DCI, Subtarget); in PerformDAGCombine()
17300 return combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget); in PerformDAGCombine()
17309 return performVFMADD_VLCombine(N, DAG, Subtarget); in PerformDAGCombine()
17316 !Subtarget.hasVInstructionsF16()) in PerformDAGCombine()
17318 return combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget); in PerformDAGCombine()
17338 MemVT.getVectorElementType().bitsLE(Subtarget.getXLenVT()) && in PerformDAGCombine()
17340 MemVT.getSizeInBits() <= Subtarget.getXLen(); in PerformDAGCombine()
17364 if (RISCVMatInt::getIntMatCost(NewC, Subtarget.getXLen(), Subtarget, in PerformDAGCombine()
17415 DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT, in PerformDAGCombine()
17429 DAG, Subtarget)) in PerformDAGCombine()
17434 if (SDValue V = performBUILD_VECTORCombine(N, DAG, Subtarget, *this)) in PerformDAGCombine()
17438 if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, *this)) in PerformDAGCombine()
17442 if (SDValue V = performINSERT_VECTOR_ELTCombine(N, DAG, Subtarget, *this)) in PerformDAGCombine()
17610 assert(Subtarget.useRVVForFixedLengthVectors()); in PerformDAGCombine()
17648 return Subtarget.hasStdExtZbb() && in shouldTransformSignedTruncationCheck()
17649 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.is64Bit()) || in shouldTransformSignedTruncationCheck()
17690 RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(), Subtarget, in isDesirableToCommuteWithShift()
17693 ShiftedC1Int, Ty.getSizeInBits(), Subtarget, in isDesirableToCommuteWithShift()
17903 const unsigned MinVLenB = Subtarget.getRealMinVLen() / 8; in computeKnownBitsForTargetNode()
17904 const unsigned MaxVLenB = Subtarget.getRealMaxVLen() / 8; in computeKnownBitsForTargetNode()
17933 uint64_t MaxVL = Subtarget.getRealMaxVLen() / SEW; in computeKnownBitsForTargetNode()
18000 unsigned XLen = Subtarget.getXLen(); in ComputeNumSignBitsForTargetNode()
18025 assert(Subtarget.getXLen() == 64); in ComputeNumSignBitsForTargetNode()
18027 assert(Subtarget.hasStdExtA()); in ComputeNumSignBitsForTargetNode()
18166 const RISCVSubtarget &Subtarget) { in emitSplitF64Pseudo() argument
18201 const RISCVSubtarget &Subtarget) { in emitBuildPairF64Pseudo() argument
18255 const RISCVSubtarget &Subtarget) { in emitQuietFCMP() argument
18292 const RISCVSubtarget &Subtarget) { in EmitLoweredCascadedSelect() argument
18328 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo(); in EmitLoweredCascadedSelect()
18394 const RISCVSubtarget &Subtarget) { in emitSelectPseudo() argument
18431 return EmitLoweredCascadedSelect(MI, *Next, BB, Subtarget); in emitSelectPseudo()
18471 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo(); in emitSelectPseudo()
18618 const RISCVSubtarget &Subtarget) { in emitFROUND() argument
18657 assert(Subtarget.is64Bit() && "Expected 64-bit GPR."); in emitFROUND()
18666 assert(Subtarget.is64Bit() && "Expected 64-bit GPR."); in emitFROUND()
18701 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo(); in emitFROUND()
18756 assert(!Subtarget.is64Bit() && in EmitInstrWithCustomInserter()
18768 return emitSelectPseudo(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
18770 return emitBuildPairF64Pseudo(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
18772 return emitSplitF64Pseudo(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
18774 return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget); in EmitInstrWithCustomInserter()
18776 return emitQuietFCMP(MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget); in EmitInstrWithCustomInserter()
18778 return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget); in EmitInstrWithCustomInserter()
18780 return emitQuietFCMP(MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget); in EmitInstrWithCustomInserter()
18782 return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget); in EmitInstrWithCustomInserter()
18784 return emitQuietFCMP(MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget); in EmitInstrWithCustomInserter()
18786 return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget); in EmitInstrWithCustomInserter()
18788 return emitQuietFCMP(MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget); in EmitInstrWithCustomInserter()
18790 return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget); in EmitInstrWithCustomInserter()
18792 return emitQuietFCMP(MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget); in EmitInstrWithCustomInserter()
18795 Subtarget); in EmitInstrWithCustomInserter()
18797 return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget); in EmitInstrWithCustomInserter()
18799 return emitQuietFCMP(MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget); in EmitInstrWithCustomInserter()
18802 Subtarget); in EmitInstrWithCustomInserter()
18823 return emitFROUND(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
18838 if (!Subtarget.is64Bit()) in EmitInstrWithCustomInserter()
19318 const RISCVSubtarget &Subtarget) { in convertLocVTToValVT() argument
19324 Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget); in convertLocVTToValVT()
19382 const RISCVSubtarget &Subtarget) { in convertValVTToLocVT() argument
19390 Val = convertToScalableVector(LocVT, Val, DAG, Subtarget); in convertValVTToLocVT()
19495 const RISCVSubtarget &Subtarget = TLI.getSubtarget(); in CC_RISCV_FastCC() local
19498 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) { in CC_RISCV_FastCC()
19510 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) { in CC_RISCV_FastCC()
19522 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) { in CC_RISCV_FastCC()
19536 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) || in CC_RISCV_FastCC()
19537 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) || in CC_RISCV_FastCC()
19538 (LocVT == MVT::f64 && Subtarget.is64Bit() && in CC_RISCV_FastCC()
19539 Subtarget.hasStdExtZdinx())) { in CC_RISCV_FastCC()
19619 const RISCVSubtarget &Subtarget = in CC_RISCV_GHC() local
19622 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) { in CC_RISCV_GHC()
19634 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) { in CC_RISCV_GHC()
19646 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) || in CC_RISCV_GHC()
19647 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() && in CC_RISCV_GHC()
19648 Subtarget.is64Bit())) { in CC_RISCV_GHC()
19677 if (Subtarget.hasStdExtE()) in LowerFormalArguments()
19679 if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx()) in LowerFormalArguments()
19699 MVT XLenVT = Subtarget.getXLenVT(); in LowerFormalArguments()
19700 unsigned XLenInBytes = Subtarget.getXLen() / 8; in LowerFormalArguments()
19761 ArrayRef<MCPhysReg> ArgRegs = RISCV::getArgGPRs(Subtarget.getTargetABI()); in LowerFormalArguments()
19868 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); in isEligibleForTailCallOptimization()
19906 MVT XLenVT = Subtarget.getXLenVT(); in LowerCall()
19915 if (Subtarget.hasStdExtE()) in LowerCall()
20054 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget); in LowerCall()
20126 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in LowerCall()
20186 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget); in LowerCall()
20270 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget); in LowerReturn()
20675 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin()) in getRegForInlineAsmConstraint()
20677 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx()) in getRegForInlineAsmConstraint()
20679 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit()) in getRegForInlineAsmConstraint()
20683 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) in getRegForInlineAsmConstraint()
20685 if (Subtarget.hasStdExtF() && VT == MVT::f32) in getRegForInlineAsmConstraint()
20687 if (Subtarget.hasStdExtD() && VT == MVT::f64) in getRegForInlineAsmConstraint()
20752 if (Subtarget.hasStdExtF()) { in getRegForInlineAsmConstraint()
20789 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) { in getRegForInlineAsmConstraint()
20796 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) { in getRegForInlineAsmConstraint()
20804 if (Subtarget.hasVInstructions()) { in getRegForInlineAsmConstraint()
20895 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT())); in LowerAsmOperandForConstraint()
20902 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT())); in LowerAsmOperandForConstraint()
20910 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT())); in LowerAsmOperandForConstraint()
20926 if (Subtarget.hasStdExtZtso()) { in emitLeadingFence()
20942 if (Subtarget.hasStdExtZtso()) { in emitTrailingFence()
20950 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Inst) && in emitTrailingFence()
20967 if (Subtarget.hasForcedAtomics()) in shouldExpandAtomicRMWInIR()
20972 if (Subtarget.hasStdExtZacas() && in shouldExpandAtomicRMWInIR()
20973 (Size >= 32 || Subtarget.hasStdExtZabha())) in shouldExpandAtomicRMWInIR()
20979 if (Size < 32 && !Subtarget.hasStdExtZabha()) in shouldExpandAtomicRMWInIR()
21055 unsigned XLen = Subtarget.getXLen(); in emitMaskedAtomicRMWIntrinsic()
21099 if (Subtarget.hasForcedAtomics()) in shouldExpandAtomicCmpXchgInIR()
21103 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) && in shouldExpandAtomicCmpXchgInIR()
21112 unsigned XLen = Subtarget.getXLen(); in emitMaskedAtomicCmpXchgIntrinsic()
21148 return Subtarget.hasStdExtZfhmin(); in shouldConvertFpToSat()
21150 return Subtarget.hasStdExtF(); in shouldConvertFpToSat()
21152 return Subtarget.hasStdExtD(); in shouldConvertFpToSat()
21161 if (Subtarget.is64Bit() && !isPositionIndependent() && in getJumpTableEncoding()
21171 assert(Subtarget.is64Bit() && !isPositionIndependent() && in LowerCustomJumpTableEntry()
21182 assert(Subtarget.getRealMinVLen() >= 64 && "zve32* unsupported"); in isVScaleKnownToBeAPowerOfTwo()
21193 if (!Subtarget.hasVendorXTHeadMemIdx()) in getIndexedAddressParts()
21251 if (Subtarget.hasVendorXCVmem()) { in getPostIndexedAddressParts()
21302 return VT.isVector() ? Subtarget.hasVInstructionsF16() in isFMAFasterThanFMulAndFAdd()
21303 : Subtarget.hasStdExtZfhOrZhinx(); in isFMAFasterThanFMulAndFAdd()
21305 return Subtarget.hasStdExtFOrZfinx(); in isFMAFasterThanFMulAndFAdd()
21307 return Subtarget.hasStdExtDOrZdinx(); in isFMAFasterThanFMulAndFAdd()
21317 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND; in getExtendForAtomicCmpSwapArg()
21333 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() && in shouldExtendTypeInLibCall()
21334 Type.getSizeInBits() < Subtarget.getXLen())) in shouldExtendTypeInLibCall()
21341 if (Subtarget.is64Bit() && Type == MVT::i32) in shouldSignExtendTypeInLibCall()
21350 const bool HasZmmul = Subtarget.hasStdExtZmmul(); in decomposeMulByConstant()
21356 if (HasZmmul && VT.getSizeInBits() > Subtarget.getXLen()) in decomposeMulByConstant()
21367 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) && in decomposeMulByConstant()
21394 if (VT.getScalarSizeInBits() > Subtarget.getXLen()) in isMulAddWithConstProfitable()
21414 *Fast = Subtarget.enableUnalignedScalarMem(); in allowsMisalignedMemoryAccesses()
21415 return Subtarget.enableUnalignedScalarMem(); in allowsMisalignedMemoryAccesses()
21431 *Fast = Subtarget.enableUnalignedVectorMem(); in allowsMisalignedMemoryAccesses()
21432 return Subtarget.enableUnalignedVectorMem(); in allowsMisalignedMemoryAccesses()
21438 if (!Subtarget.hasVInstructions()) in getOptimalMemOpType()
21452 const unsigned MinVLenInBytes = Subtarget.getRealMinVLen()/8; in getOptimalMemOpType()
21462 MVT ELenVT = MVT::getIntegerVT(Subtarget.getELen()); in getOptimalMemOpType()
21467 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) { in getOptimalMemOpType()
21607 if (Subtarget.isTargetFuchsia()) in getIRStackGuard()
21613 if (Subtarget.isTargetAndroid()) in getIRStackGuard()
21635 if (!Subtarget.useRVVForFixedLengthVectors()) in isLegalInterleavedAccessType()
21659 if (!Subtarget.hasVInstructions()) in isLegalStridedLoadStore()
21663 if (DataType.isFixedLengthVector() && !Subtarget.useRVVForFixedLengthVectors()) in isLegalStridedLoadStore()
21670 if (!Subtarget.enableUnalignedVectorMem() && in isLegalStridedLoadStore()
21706 auto *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen()); in lowerInterleavedLoad()
21760 auto *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen()); in lowerInterleavedStore()
21807 Type *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen()); in lowerDeinterleaveIntrinsicToLoad()
21857 Type *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen()); in lowerInterleaveIntrinsicToStore()
21912 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF); in getRegisterByName()
21913 if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg)) in getRegisterByName()
21971 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb(); in isCtpopFast()
21972 if (VT.isFixedLengthVector() && Subtarget.hasStdExtZvbb()) in isCtpopFast()
21974 return Subtarget.hasStdExtZbb() && in isCtpopFast()
22019 if (!Subtarget.hasShortForwardBranchOpt()) in BuildSDIVPow2()
22022 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit()))) in BuildSDIVPow2()
22033 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) in shouldFoldSelectWithSingleBitTest()
22034 return !Subtarget.hasStdExtZbs() && AndMask.ugt(1024); in shouldFoldSelectWithSingleBitTest()
22039 return Subtarget.getMinimumJumpTableEntries(); in getMinimumJumpTableEntries()
22221 if (Subtarget.hasStdExtZicfilp()) { in expandIndirectJTBranch()