Lines Matching refs:MVT
88 addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); in SITargetLowering()
89 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); in SITargetLowering()
91 addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass); in SITargetLowering()
92 addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); in SITargetLowering()
94 addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass); in SITargetLowering()
99 addRegisterClass(MVT::f64, V64RegClass); in SITargetLowering()
100 addRegisterClass(MVT::v2f32, V64RegClass); in SITargetLowering()
101 addRegisterClass(MVT::Untyped, V64RegClass); in SITargetLowering()
103 addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); in SITargetLowering()
104 addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); in SITargetLowering()
106 addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
107 addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
109 addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
110 addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128)); in SITargetLowering()
112 addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass); in SITargetLowering()
113 addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160)); in SITargetLowering()
115 addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass); in SITargetLowering()
116 addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192)); in SITargetLowering()
118 addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass); in SITargetLowering()
119 addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192)); in SITargetLowering()
121 addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass); in SITargetLowering()
122 addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224)); in SITargetLowering()
124 addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
125 addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256)); in SITargetLowering()
127 addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
128 addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256)); in SITargetLowering()
130 addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass); in SITargetLowering()
131 addRegisterClass(MVT::v9f32, TRI->getVGPRClassForBitWidth(288)); in SITargetLowering()
133 addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass); in SITargetLowering()
134 addRegisterClass(MVT::v10f32, TRI->getVGPRClassForBitWidth(320)); in SITargetLowering()
136 addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass); in SITargetLowering()
137 addRegisterClass(MVT::v11f32, TRI->getVGPRClassForBitWidth(352)); in SITargetLowering()
139 addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass); in SITargetLowering()
140 addRegisterClass(MVT::v12f32, TRI->getVGPRClassForBitWidth(384)); in SITargetLowering()
142 addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
143 addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512)); in SITargetLowering()
145 addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
146 addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512)); in SITargetLowering()
148 addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass); in SITargetLowering()
149 addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024)); in SITargetLowering()
153 addRegisterClass(MVT::i16, &AMDGPU::VGPR_16RegClass); in SITargetLowering()
154 addRegisterClass(MVT::f16, &AMDGPU::VGPR_16RegClass); in SITargetLowering()
155 addRegisterClass(MVT::bf16, &AMDGPU::VGPR_16RegClass); in SITargetLowering()
157 addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
158 addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
159 addRegisterClass(MVT::bf16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
163 addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
164 addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
165 addRegisterClass(MVT::v2bf16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
166 addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
167 addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
168 addRegisterClass(MVT::v4bf16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
169 addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
170 addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
171 addRegisterClass(MVT::v8bf16, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
172 addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
173 addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
174 addRegisterClass(MVT::v16bf16, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
175 addRegisterClass(MVT::v32i16, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
176 addRegisterClass(MVT::v32f16, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
177 addRegisterClass(MVT::v32bf16, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
180 addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass); in SITargetLowering()
181 addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024)); in SITargetLowering()
194 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, in SITargetLowering()
195 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32, in SITargetLowering()
196 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32, in SITargetLowering()
197 MVT::i1, MVT::v32i32}, in SITargetLowering()
201 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, in SITargetLowering()
202 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32, in SITargetLowering()
203 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32, in SITargetLowering()
204 MVT::i1, MVT::v32i32}, in SITargetLowering()
207 if (isTypeLegal(MVT::bf16)) { in SITargetLowering()
219 setOperationAction(Opc, MVT::bf16, Promote); in SITargetLowering()
220 AddPromotedToType(Opc, MVT::bf16, MVT::f32); in SITargetLowering()
223 setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand); in SITargetLowering()
225 setOperationAction(ISD::SELECT, MVT::bf16, Promote); in SITargetLowering()
226 AddPromotedToType(ISD::SELECT, MVT::bf16, MVT::i16); in SITargetLowering()
228 setOperationAction(ISD::FABS, MVT::bf16, Legal); in SITargetLowering()
229 setOperationAction(ISD::FNEG, MVT::bf16, Legal); in SITargetLowering()
230 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Legal); in SITargetLowering()
234 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in SITargetLowering()
235 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in SITargetLowering()
238 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); in SITargetLowering()
239 setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand); in SITargetLowering()
240 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); in SITargetLowering()
241 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand); in SITargetLowering()
242 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand); in SITargetLowering()
243 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand); in SITargetLowering()
244 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand); in SITargetLowering()
245 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand); in SITargetLowering()
246 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand); in SITargetLowering()
247 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand); in SITargetLowering()
248 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand); in SITargetLowering()
249 setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand); in SITargetLowering()
250 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand); in SITargetLowering()
251 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); in SITargetLowering()
252 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand); in SITargetLowering()
253 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand); in SITargetLowering()
255 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); in SITargetLowering()
256 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); in SITargetLowering()
257 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand); in SITargetLowering()
258 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand); in SITargetLowering()
259 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand); in SITargetLowering()
260 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand); in SITargetLowering()
261 setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand); in SITargetLowering()
263 setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom); in SITargetLowering()
265 setOperationAction(ISD::SELECT, MVT::i1, Promote); in SITargetLowering()
266 setOperationAction(ISD::SELECT, MVT::i64, Custom); in SITargetLowering()
267 setOperationAction(ISD::SELECT, MVT::f64, Promote); in SITargetLowering()
268 AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64); in SITargetLowering()
270 setOperationAction(ISD::FSQRT, {MVT::f32, MVT::f64}, Custom); in SITargetLowering()
273 {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand); in SITargetLowering()
275 setOperationAction(ISD::SETCC, MVT::i1, Promote); in SITargetLowering()
276 setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand); in SITargetLowering()
277 AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32); in SITargetLowering()
280 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, in SITargetLowering()
281 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32, in SITargetLowering()
282 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32}, in SITargetLowering()
285 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, in SITargetLowering()
286 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32, in SITargetLowering()
287 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32}, in SITargetLowering()
291 {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16, in SITargetLowering()
292 MVT::v3i16, MVT::v4i16, MVT::Other}, in SITargetLowering()
295 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in SITargetLowering()
297 {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand); in SITargetLowering()
299 setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Legal); in SITargetLowering()
301 setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i32, Legal); in SITargetLowering()
303 setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64, in SITargetLowering()
307 setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal); in SITargetLowering()
312 for (MVT VT : in SITargetLowering()
313 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32, in SITargetLowering()
314 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32, in SITargetLowering()
315 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16, in SITargetLowering()
316 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32, in SITargetLowering()
317 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, in SITargetLowering()
318 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16, in SITargetLowering()
319 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32, in SITargetLowering()
320 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) { in SITargetLowering()
345 setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand); in SITargetLowering()
352 for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) { in SITargetLowering()
354 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
357 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
360 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
363 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
366 for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) { in SITargetLowering()
368 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32); in SITargetLowering()
371 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32); in SITargetLowering()
374 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32); in SITargetLowering()
377 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32); in SITargetLowering()
380 for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) { in SITargetLowering()
382 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32); in SITargetLowering()
385 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32); in SITargetLowering()
388 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32); in SITargetLowering()
391 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32); in SITargetLowering()
394 for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) { in SITargetLowering()
396 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32); in SITargetLowering()
399 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32); in SITargetLowering()
402 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32); in SITargetLowering()
405 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32); in SITargetLowering()
408 for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) { in SITargetLowering()
410 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32); in SITargetLowering()
413 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32); in SITargetLowering()
416 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32); in SITargetLowering()
419 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32); in SITargetLowering()
423 {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32}, in SITargetLowering()
426 setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16}, in SITargetLowering()
432 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8, in SITargetLowering()
433 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16}, in SITargetLowering()
438 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom); in SITargetLowering()
442 {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32, in SITargetLowering()
443 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32, in SITargetLowering()
444 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32, in SITargetLowering()
445 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32}, in SITargetLowering()
450 setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom); in SITargetLowering()
454 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, {MVT::i32, MVT::i64}, in SITargetLowering()
457 setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom); in SITargetLowering()
459 setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal); in SITargetLowering()
464 setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal); in SITargetLowering()
467 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in SITargetLowering()
471 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Legal); in SITargetLowering()
472 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Custom); in SITargetLowering()
475 setOperationAction({ISD::FPOW, ISD::FPOWI}, MVT::f16, Promote); in SITargetLowering()
476 setOperationAction({ISD::FLOG, ISD::FEXP, ISD::FLOG10}, MVT::f16, Custom); in SITargetLowering()
478 setOperationAction(ISD::FSQRT, MVT::f16, Custom); in SITargetLowering()
482 setOperationAction(ISD::FMAD, MVT::f32, Legal); in SITargetLowering()
486 setOperationAction(ISD::FCOPYSIGN, {MVT::f32, MVT::f64}, Expand); in SITargetLowering()
489 setOperationAction(ISD::CTPOP, MVT::i32, Expand); in SITargetLowering()
492 setOperationAction(ISD::CTPOP, MVT::i64, Expand); in SITargetLowering()
495 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom); in SITargetLowering()
498 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom); in SITargetLowering()
513 setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Legal); in SITargetLowering()
516 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32}, in SITargetLowering()
519 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, {MVT::f32, MVT::f64}, in SITargetLowering()
526 {MVT::f32, MVT::f64}, Legal); in SITargetLowering()
529 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, MVT::f64, in SITargetLowering()
533 MVT::f64, Custom); in SITargetLowering()
535 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); in SITargetLowering()
536 setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, {MVT::f32, MVT::f64}, in SITargetLowering()
538 setOperationAction(ISD::FFREXP, {MVT::f32, MVT::f64}, Custom); in SITargetLowering()
540 setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FDIV}, MVT::f32, Custom); in SITargetLowering()
541 setOperationAction(ISD::FDIV, MVT::f64, Custom); in SITargetLowering()
543 setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand); in SITargetLowering()
544 setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand); in SITargetLowering()
548 setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f32, Custom); in SITargetLowering()
549 setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f64, Custom); in SITargetLowering()
554 MVT::i16, Legal); in SITargetLowering()
556 AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32); in SITargetLowering()
559 MVT::i16, Expand); in SITargetLowering()
565 MVT::i16, Promote); in SITargetLowering()
567 setOperationAction(ISD::LOAD, MVT::i16, Custom); in SITargetLowering()
569 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in SITargetLowering()
571 setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote); in SITargetLowering()
572 AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32); in SITargetLowering()
573 setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote); in SITargetLowering()
574 AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32); in SITargetLowering()
576 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom); in SITargetLowering()
577 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom); in SITargetLowering()
578 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom); in SITargetLowering()
580 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i32, Custom); in SITargetLowering()
583 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); in SITargetLowering()
584 setOperationAction(ISD::ConstantFP, MVT::bf16, Legal); in SITargetLowering()
587 setOperationAction(ISD::LOAD, MVT::f16, Promote); in SITargetLowering()
588 AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16); in SITargetLowering()
589 setOperationAction(ISD::STORE, MVT::f16, Promote); in SITargetLowering()
590 AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16); in SITargetLowering()
593 setOperationAction(ISD::LOAD, MVT::bf16, Promote); in SITargetLowering()
594 AddPromotedToType(ISD::LOAD, MVT::bf16, MVT::i16); in SITargetLowering()
595 setOperationAction(ISD::STORE, MVT::bf16, Promote); in SITargetLowering()
596 AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16); in SITargetLowering()
601 MVT::f16, Custom); in SITargetLowering()
603 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::f16, Promote); in SITargetLowering()
604 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::bf16, Promote); in SITargetLowering()
607 setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, {MVT::f16, MVT::bf16}, in SITargetLowering()
609 setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, MVT::f16, Custom); in SITargetLowering()
610 setOperationAction(ISD::FFREXP, MVT::f16, Custom); in SITargetLowering()
611 setOperationAction(ISD::FDIV, MVT::f16, Custom); in SITargetLowering()
614 setOperationAction(ISD::FMA, MVT::f16, Legal); in SITargetLowering()
616 setOperationAction(ISD::FMAD, MVT::f16, Legal); in SITargetLowering()
618 for (MVT VT : in SITargetLowering()
619 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16, in SITargetLowering()
620 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, in SITargetLowering()
621 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) { in SITargetLowering()
647 setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal); in SITargetLowering()
648 setOperationAction(ISD::BSWAP, MVT::v4i16, Custom); in SITargetLowering()
651 setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal); in SITargetLowering()
653 setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, in SITargetLowering()
656 setOperationAction(ISD::STORE, MVT::v2i16, Promote); in SITargetLowering()
657 AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32); in SITargetLowering()
658 setOperationAction(ISD::STORE, MVT::v2f16, Promote); in SITargetLowering()
659 AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32); in SITargetLowering()
661 setOperationAction(ISD::LOAD, MVT::v2i16, Promote); in SITargetLowering()
662 AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32); in SITargetLowering()
663 setOperationAction(ISD::LOAD, MVT::v2f16, Promote); in SITargetLowering()
664 AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); in SITargetLowering()
666 setOperationAction(ISD::AND, MVT::v2i16, Promote); in SITargetLowering()
667 AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); in SITargetLowering()
668 setOperationAction(ISD::OR, MVT::v2i16, Promote); in SITargetLowering()
669 AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32); in SITargetLowering()
670 setOperationAction(ISD::XOR, MVT::v2i16, Promote); in SITargetLowering()
671 AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32); in SITargetLowering()
673 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); in SITargetLowering()
674 AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); in SITargetLowering()
675 setOperationAction(ISD::LOAD, MVT::v4f16, Promote); in SITargetLowering()
676 AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); in SITargetLowering()
677 setOperationAction(ISD::LOAD, MVT::v4bf16, Promote); in SITargetLowering()
678 AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32); in SITargetLowering()
680 setOperationAction(ISD::STORE, MVT::v4i16, Promote); in SITargetLowering()
681 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); in SITargetLowering()
682 setOperationAction(ISD::STORE, MVT::v4f16, Promote); in SITargetLowering()
683 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); in SITargetLowering()
684 setOperationAction(ISD::STORE, MVT::v4bf16, Promote); in SITargetLowering()
685 AddPromotedToType(ISD::STORE, MVT::v4bf16, MVT::v2i32); in SITargetLowering()
687 setOperationAction(ISD::LOAD, MVT::v8i16, Promote); in SITargetLowering()
688 AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32); in SITargetLowering()
689 setOperationAction(ISD::LOAD, MVT::v8f16, Promote); in SITargetLowering()
690 AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32); in SITargetLowering()
691 setOperationAction(ISD::LOAD, MVT::v8bf16, Promote); in SITargetLowering()
692 AddPromotedToType(ISD::LOAD, MVT::v8bf16, MVT::v4i32); in SITargetLowering()
694 setOperationAction(ISD::STORE, MVT::v4i16, Promote); in SITargetLowering()
695 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); in SITargetLowering()
696 setOperationAction(ISD::STORE, MVT::v4f16, Promote); in SITargetLowering()
697 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); in SITargetLowering()
699 setOperationAction(ISD::STORE, MVT::v8i16, Promote); in SITargetLowering()
700 AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32); in SITargetLowering()
701 setOperationAction(ISD::STORE, MVT::v8f16, Promote); in SITargetLowering()
702 AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32); in SITargetLowering()
703 setOperationAction(ISD::STORE, MVT::v8bf16, Promote); in SITargetLowering()
704 AddPromotedToType(ISD::STORE, MVT::v8bf16, MVT::v4i32); in SITargetLowering()
706 setOperationAction(ISD::LOAD, MVT::v16i16, Promote); in SITargetLowering()
707 AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32); in SITargetLowering()
708 setOperationAction(ISD::LOAD, MVT::v16f16, Promote); in SITargetLowering()
709 AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32); in SITargetLowering()
710 setOperationAction(ISD::LOAD, MVT::v16bf16, Promote); in SITargetLowering()
711 AddPromotedToType(ISD::LOAD, MVT::v16bf16, MVT::v8i32); in SITargetLowering()
713 setOperationAction(ISD::STORE, MVT::v16i16, Promote); in SITargetLowering()
714 AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32); in SITargetLowering()
715 setOperationAction(ISD::STORE, MVT::v16f16, Promote); in SITargetLowering()
716 AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32); in SITargetLowering()
717 setOperationAction(ISD::STORE, MVT::v16bf16, Promote); in SITargetLowering()
718 AddPromotedToType(ISD::STORE, MVT::v16bf16, MVT::v8i32); in SITargetLowering()
720 setOperationAction(ISD::LOAD, MVT::v32i16, Promote); in SITargetLowering()
721 AddPromotedToType(ISD::LOAD, MVT::v32i16, MVT::v16i32); in SITargetLowering()
722 setOperationAction(ISD::LOAD, MVT::v32f16, Promote); in SITargetLowering()
723 AddPromotedToType(ISD::LOAD, MVT::v32f16, MVT::v16i32); in SITargetLowering()
724 setOperationAction(ISD::LOAD, MVT::v32bf16, Promote); in SITargetLowering()
725 AddPromotedToType(ISD::LOAD, MVT::v32bf16, MVT::v16i32); in SITargetLowering()
727 setOperationAction(ISD::STORE, MVT::v32i16, Promote); in SITargetLowering()
728 AddPromotedToType(ISD::STORE, MVT::v32i16, MVT::v16i32); in SITargetLowering()
729 setOperationAction(ISD::STORE, MVT::v32f16, Promote); in SITargetLowering()
730 AddPromotedToType(ISD::STORE, MVT::v32f16, MVT::v16i32); in SITargetLowering()
731 setOperationAction(ISD::STORE, MVT::v32bf16, Promote); in SITargetLowering()
732 AddPromotedToType(ISD::STORE, MVT::v32bf16, MVT::v16i32); in SITargetLowering()
735 MVT::v2i32, Expand); in SITargetLowering()
736 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand); in SITargetLowering()
739 MVT::v4i32, Expand); in SITargetLowering()
742 MVT::v8i32, Expand); in SITargetLowering()
744 setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, in SITargetLowering()
747 setOperationAction(ISD::FNEG, MVT::v2f16, Legal); in SITargetLowering()
750 setOperationAction(ISD::FABS, MVT::v2f16, Legal); in SITargetLowering()
752 setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, MVT::f16, Custom); in SITargetLowering()
753 setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal); in SITargetLowering()
756 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, in SITargetLowering()
760 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, in SITargetLowering()
763 for (MVT Vec16 : in SITargetLowering()
764 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16, in SITargetLowering()
765 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) { in SITargetLowering()
777 MVT::v2i16, Legal); in SITargetLowering()
781 MVT::v2f16, Legal); in SITargetLowering()
783 setOperationAction(ISD::EXTRACT_VECTOR_ELT, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, in SITargetLowering()
787 {MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16, in SITargetLowering()
788 MVT::v16f16, MVT::v16i16, MVT::v32f16, MVT::v32i16}, in SITargetLowering()
791 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16}) in SITargetLowering()
799 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}) in SITargetLowering()
804 setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, {MVT::v2f16, MVT::v4f16}, in SITargetLowering()
807 setOperationAction(ISD::FEXP, MVT::v2f16, Custom); in SITargetLowering()
808 setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16}, in SITargetLowering()
813 MVT::v2f32, Legal); in SITargetLowering()
815 {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32}, in SITargetLowering()
820 setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v4f16, Custom); in SITargetLowering()
823 setOperationAction(ISD::SELECT, MVT::v2i16, Promote); in SITargetLowering()
824 AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32); in SITargetLowering()
825 setOperationAction(ISD::SELECT, MVT::v2f16, Promote); in SITargetLowering()
826 AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32); in SITargetLowering()
829 setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom); in SITargetLowering()
831 setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v2f16, Custom); in SITargetLowering()
835 {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8, in SITargetLowering()
836 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16, in SITargetLowering()
837 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16, in SITargetLowering()
838 MVT::v32f16, MVT::v32bf16}, in SITargetLowering()
841 setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom); in SITargetLowering()
844 setOperationAction(ISD::MUL, MVT::i64, Custom); in SITargetLowering()
847 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom); in SITargetLowering()
850 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in SITargetLowering()
854 {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal); in SITargetLowering()
856 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, in SITargetLowering()
861 {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16, in SITargetLowering()
862 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128, in SITargetLowering()
863 MVT::i8}, in SITargetLowering()
867 {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16, in SITargetLowering()
868 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16, in SITargetLowering()
869 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16, in SITargetLowering()
870 MVT::i16, MVT::bf16, MVT::i8, MVT::i128}, in SITargetLowering()
874 {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16, in SITargetLowering()
875 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16, in SITargetLowering()
876 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, in SITargetLowering()
877 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128}, in SITargetLowering()
880 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); in SITargetLowering()
881 setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom); in SITargetLowering()
882 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); in SITargetLowering()
883 setOperationAction(ISD::GET_FPENV, MVT::i64, Custom); in SITargetLowering()
884 setOperationAction(ISD::SET_FPENV, MVT::i64, Custom); in SITargetLowering()
888 setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal); in SITargetLowering()
890 setOperationAction(ISD::MUL, MVT::i1, Promote); in SITargetLowering()
982 DestVT.getScalarType() == MVT::f32 && in isFPExtFoldable()
983 SrcVT.getScalarType() == MVT::f16 && in isFPExtFoldable()
1004 MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, in getRegisterTypeForCallingConv()
1016 return MVT::v2i16; in getRegisterTypeForCallingConv()
1017 return (ScalarVT == MVT::bf16 ? MVT::i32 : MVT::v2f16); in getRegisterTypeForCallingConv()
1019 return VT.isInteger() ? MVT::i32 : MVT::f32; in getRegisterTypeForCallingConv()
1023 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32; in getRegisterTypeForCallingConv()
1024 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32; in getRegisterTypeForCallingConv()
1028 return MVT::i32; in getRegisterTypeForCallingConv()
1062 unsigned &NumIntermediates, MVT &RegisterVT) const { in getVectorTypeBreakdownForCallingConv()
1071 if (ScalarVT == MVT::bf16) { in getVectorTypeBreakdownForCallingConv()
1072 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1073 IntermediateVT = MVT::v2bf16; in getVectorTypeBreakdownForCallingConv()
1075 RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16; in getVectorTypeBreakdownForCallingConv()
1091 RegisterVT = MVT::i16; in getVectorTypeBreakdownForCallingConv()
1099 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1106 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1153 MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const { in getPointerTy()
1155 return MVT::v5i32; in getPointerTy()
1158 return MVT::v6i32; in getPointerTy()
1165 MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const { in getPointerMemTy()
1170 return MVT::v8i32; in getPointerMemTy()
1264 Info.memVT = MVT::i32; in getTgtMemIntrinsic()
1268 Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType()); in getTgtMemIntrinsic()
1297 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1311 Info.memVT = MVT::getVT(CI.getOperand(0)->getType()); in getTgtMemIntrinsic()
1320 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1333 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1343 Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT? in getTgtMemIntrinsic()
1366 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1378 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1399 Info.memVT = MVT::i32; in getTgtMemIntrinsic()
1427 Info.memVT = MVT::i32; in getTgtMemIntrinsic()
1447 Ops.push_back(DAG.getTargetConstant(SrcAS, SDLoc(), MVT::i32)); in CollectTargetIntrinsicOperands()
1448 Ops.push_back(DAG.getTargetConstant(DstAS, SDLoc(), MVT::i32)); in CollectTargetIntrinsicOperands()
1877 return MVT::v4i32; in getOptimalMemOpType()
1880 return MVT::v2i32; in getOptimalMemOpType()
1883 return MVT::Other; in getOptimalMemOpType()
1915 SITargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
1917 VT.getScalarType().bitsLE(MVT::i16)) in getPreferredVectorAction()
1938 if (Subtarget->has16BitInsts() && VT == MVT::i16) { in isTypeDesirableForOp()
1957 if (VT == MVT::i1 && Op == ISD::SETCC) in isTypeDesirableForOp()
1974 MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); in lowerKernArgParameterPtr()
2005 return DAG.getConstant(*KnownSize, SL, MVT::i32); in getLDSKernelId()
2020 DAG.getConstant(0, SL, MVT::i32)); in convertArgType()
2059 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), in lowerKernargMemParameter()
2063 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
2064 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
2092 return DAG.getFrameIndex(FrameIdx, MVT::i32); in lowerStackParameter()
2101 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); in lowerStackParameter()
2106 MVT MemVT = VA.getValVT(); in lowerStackParameter()
2925 MVT VT = VA.getLocVT(); in LowerFormalArguments()
2965 SDValue Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32); in LowerFormalArguments()
2967 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, DL, MVT::i32); in LowerFormalArguments()
2968 SDValue Extract = DAG.getNode(ISD::SRL, DL, MVT::i32, Copy, ShiftAmt); in LowerFormalArguments()
3000 Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32); in LowerFormalArguments()
3004 DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i32, in LowerFormalArguments()
3042 DAG.getValueType(MVT::i16)); in LowerFormalArguments()
3127 DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in LowerFormalArguments()
3233 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); in LowerReturn()
3235 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); in LowerReturn()
3250 return DAG.getNode(Opc, DL, MVT::Other, RetOps); in LowerReturn()
3380 EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32; in passSpecialInputs()
3451 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX); in passSpecialInputs()
3453 InputReg = DAG.getConstant(0, DL, MVT::i32); in passSpecialInputs()
3459 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY); in passSpecialInputs()
3460 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, in passSpecialInputs()
3461 DAG.getShiftAmountConstant(10, MVT::i32, SL)); in passSpecialInputs()
3463 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; in passSpecialInputs()
3468 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ); in passSpecialInputs()
3469 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, in passSpecialInputs()
3470 DAG.getShiftAmountConstant(20, MVT::i32, SL)); in passSpecialInputs()
3472 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; in passSpecialInputs()
3481 InputReg = DAG.getUNDEF(MVT::i32); in passSpecialInputs()
3489 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg); in passSpecialInputs()
3742 = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32); in LowerCall()
3752 MVT PtrVT = MVT::i32; in LowerCall()
3824 MVT::i32); in LowerCall()
3825 DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff); in LowerCall()
3833 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32); in LowerCall()
3851 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
3879 Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64)); in LowerCall()
3881 Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); in LowerCall()
3888 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); in LowerCall()
3914 MVT::Glue, GlueOps), in LowerCall()
3921 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
3988 DAG.getConstant(Subtarget->getWavefrontSizeLog2(), dl, MVT::i32)); in lowerDYNAMIC_STACKALLOCImpl()
4020 if (Op.getValueType() != MVT::i32) in LowerSTACKSAVE()
4026 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32); in LowerSTACKSAVE()
4032 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP); in LowerSTACKSAVE()
4039 assert(Op.getValueType() == MVT::i32); in lowerGET_ROUNDING()
4043 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32); in lowerGET_ROUNDING()
4046 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); in lowerGET_ROUNDING()
4079 DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64); in lowerGET_ROUNDING()
4081 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerGET_ROUNDING()
4083 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two); in lowerGET_ROUNDING()
4088 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits); in lowerGET_ROUNDING()
4089 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue); in lowerGET_ROUNDING()
4091 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32); in lowerGET_ROUNDING()
4093 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask); in lowerGET_ROUNDING()
4097 SDValue Four = DAG.getConstant(4, SL, MVT::i32); in lowerGET_ROUNDING()
4099 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT); in lowerGET_ROUNDING()
4100 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four); in lowerGET_ROUNDING()
4101 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue, in lowerGET_ROUNDING()
4112 assert(NewMode.getValueType() == MVT::i32); in lowerSET_ROUNDING()
4121 AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32); in lowerSET_ROUNDING()
4133 AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32); in lowerSET_ROUNDING()
4135 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerSET_ROUNDING()
4137 DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two); in lowerSET_ROUNDING()
4140 DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits); in lowerSET_ROUNDING()
4148 DAG.getConstant(AMDGPU::FltRoundToHWConversionTable, SL, MVT::i64); in lowerSET_ROUNDING()
4150 SDValue Four = DAG.getConstant(4, SL, MVT::i32); in lowerSET_ROUNDING()
4151 SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four); in lowerSET_ROUNDING()
4153 DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum); in lowerSET_ROUNDING()
4155 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerSET_ROUNDING()
4157 DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two); in lowerSET_ROUNDING()
4160 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits); in lowerSET_ROUNDING()
4161 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue); in lowerSET_ROUNDING()
4172 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32); in lowerSET_ROUNDING()
4173 NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, in lowerSET_ROUNDING()
4180 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32); in lowerSET_ROUNDING()
4183 SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32); in lowerSET_ROUNDING()
4215 if (SrcVT.getScalarType() != MVT::bf16) in lowerFP_EXTEND()
4231 if (Op.getValueType() != MVT::i64) in lowerGET_FPENV()
4236 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32); in lowerGET_FPENV()
4239 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32); in lowerGET_FPENV()
4241 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::Other); in lowerGET_FPENV()
4243 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); in lowerGET_FPENV()
4249 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1), in lowerGET_FPENV()
4253 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg); in lowerGET_FPENV()
4254 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); in lowerGET_FPENV()
4261 if (Op.getOperand(1).getValueType() != MVT::i64) in lowerSET_FPENV()
4264 SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1)); in lowerSET_FPENV()
4265 SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input, in lowerSET_FPENV()
4266 DAG.getConstant(0, SL, MVT::i32)); in lowerSET_FPENV()
4267 SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input, in lowerSET_FPENV()
4268 DAG.getConstant(1, SL, MVT::i32)); in lowerSET_FPENV()
4271 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32); in lowerSET_FPENV()
4272 NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, in lowerSET_FPENV()
4274 NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, in lowerSET_FPENV()
4279 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32); in lowerSET_FPENV()
4282 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32); in lowerSET_FPENV()
4285 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32); in lowerSET_FPENV()
4287 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0), in lowerSET_FPENV()
4290 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0), in lowerSET_FPENV()
4292 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg); in lowerSET_FPENV()
5559 return MVT::i1; in getSetCCResultType()
5561 return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements()); in getSetCCResultType()
5564 MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { in getScalarShiftAmountTy()
5567 return (VT == MVT::i16) ? MVT::i16 : MVT::i32; in getScalarShiftAmountTy()
5596 case MVT::f32: { in isFMAFasterThanFMulAndFAdd()
5610 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
5612 case MVT::f16: in isFMAFasterThanFMulAndFAdd()
5625 return isFMAFasterThanFMulAndFAdd(MF, MVT::f16); in isFMAFasterThanFMulAndFAdd()
5627 return isFMAFasterThanFMulAndFAdd(MF, MVT::f32); in isFMAFasterThanFMulAndFAdd()
5629 return isFMAFasterThanFMulAndFAdd(MF, MVT::f64); in isFMAFasterThanFMulAndFAdd()
5655 if (VT == MVT::f32) in isFMADLegal()
5658 if (VT == MVT::f16) { in isFMADLegal()
5676 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || in splitUnaryVectorOp()
5677 VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 || in splitUnaryVectorOp()
5678 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || in splitUnaryVectorOp()
5679 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16); in splitUnaryVectorOp()
5699 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || in splitBinaryVectorOp()
5700 VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 || in splitBinaryVectorOp()
5701 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || in splitBinaryVectorOp()
5702 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16); in splitBinaryVectorOp()
5723 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v8i16 || in splitTernaryVectorOp()
5724 VT == MVT::v8f16 || VT == MVT::v4f32 || VT == MVT::v16i16 || in splitTernaryVectorOp()
5725 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || in splitTernaryVectorOp()
5726 VT == MVT::v32f32 || VT == MVT::v32f16 || VT == MVT::v32i16 || in splitTernaryVectorOp()
5727 VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v16bf16 || in splitTernaryVectorOp()
5728 VT == MVT::v32bf16); in splitTernaryVectorOp()
5766 if (VT == MVT::f32) in LowerOperation()
5768 if (VT == MVT::f64) in LowerOperation()
5808 if (Op.getOperand(0)->getValueType(0) != MVT::f32) in LowerOperation()
5919 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); in adjustLoadValueTypeImpl()
5923 Elts.push_back(DAG.getUNDEF(MVT::i16)); in adjustLoadValueTypeImpl()
5948 EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in adjustLoadValueType()
5959 SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other); in adjustLoadValueType()
6005 SDVTList VTList = DAG.getVTList(CastVT, MVT::Other); in lowerIntrinsicLoad()
6028 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) { in lowerICMPIntrinsic()
6031 LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS); in lowerICMPIntrinsic()
6032 RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS); in lowerICMPIntrinsic()
6060 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) { in lowerFCMPIntrinsic()
6061 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
6062 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
6109 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), in lowerBALLOTIntrinsic()
6110 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); in lowerBALLOTIntrinsic()
6121 MVT IntVT = MVT::getIntegerVT(ValSize); in lowerLaneOp()
6124 SDValue Src2, MVT ValT) -> SDValue { in lowerLaneOp()
6147 Operands.push_back(DAG.getTargetConstant(IID, SL, MVT::i32)); in lowerLaneOp()
6153 Operands.push_back(DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue, in lowerLaneOp()
6177 SL, MVT::i32); in lowerLaneOp()
6181 SL, MVT::i32); in lowerLaneOp()
6186 SL, MVT::i32); in lowerLaneOp()
6189 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32); in lowerLaneOp()
6227 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue, in lowerLaneOp()
6238 switch (MVT::SimpleValueType EltTy = in lowerLaneOp()
6240 case MVT::i32: in lowerLaneOp()
6241 case MVT::f32: { in lowerLaneOp()
6245 case MVT::i16: in lowerLaneOp()
6246 case MVT::f16: in lowerLaneOp()
6247 case MVT::bf16: { in lowerLaneOp()
6248 MVT SubVecVT = MVT::getVectorVT(EltTy, 2); in lowerLaneOp()
6253 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerLaneOp()
6257 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerLaneOp()
6261 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerLaneOp()
6277 MVT VecVT = MVT::getVectorVT(MVT::i32, ValSize / 32); in lowerLaneOp()
6315 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
6317 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
6342 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
6343 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
6360 assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n"); in ReplaceNodeResults()
6377 DAG.getVTList(MVT::i32), Ops, VT, MMO); in ReplaceNodeResults()
6383 DAG.getConstant(0, DL, MVT::i32), // vindex in ReplaceNodeResults()
6388 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in ReplaceNodeResults()
6423 if (NewVT.bitsLT(MVT::i32)) { in ReplaceNodeResults()
6424 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
6425 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
6426 SelectVT = MVT::i32; in ReplaceNodeResults()
6438 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
6442 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6444 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
6446 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
6447 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6451 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
6455 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6457 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
6459 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
6460 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6464 if (N->getValueType(0) != MVT::f16) in ReplaceNodeResults()
6638 MVT VT = Op.getSimpleValueType(); in LowerRETURNADDR()
6668 DAG.getTargetConstant(0, DL, MVT::i32)); in getFPExtOrFPRound()
6672 assert(Op.getValueType() == MVT::f16 && in lowerFP_ROUND()
6677 if (SrcVT != MVT::f64) in lowerFP_ROUND()
6686 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); in lowerFP_ROUND()
6687 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); in lowerFP_ROUND()
6688 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); in lowerFP_ROUND()
6705 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 || in lowerFMINNUM_FMAXNUM()
6706 VT == MVT::v16bf16) in lowerFMINNUM_FMAXNUM()
6714 assert(VT == MVT::f16); in lowerFLDEXP()
6718 if (ExpVT == MVT::i16) in lowerFLDEXP()
6735 SDValue TruncExp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Clamp); in lowerFLDEXP()
6738 return DAG.getNode(ISD::STRICT_FLDEXP, DL, {VT, MVT::Other}, in lowerFLDEXP()
6753 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64"); in lowerMUL()
6818 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); in lowerXMULO()
6820 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, in lowerXMULO()
6834 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) in lowerXMULO()
6836 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); in lowerXMULO()
6869 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain); in lowerTrapEndpgm()
6872 SDValue SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT, in loadImplicitKernelArgument()
6893 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR); in lowerTrapHsaQueuePtr()
6903 QueuePtr = DAG.getConstant(0, SL, MVT::i64); in lowerTrapHsaQueuePtr()
6906 MVT::i64); in lowerTrapHsaQueuePtr()
6910 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); in lowerTrapHsaQueuePtr()
6917 DAG.getTargetConstant(TrapID, SL, MVT::i16), in lowerTrapHsaQueuePtr()
6921 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsaQueuePtr()
6932 return DAG.getNode(AMDGPUISD::SIMULATED_TRAP, SL, MVT::Other, Chain); in lowerTrapHsa()
6937 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerTrapHsa()
6939 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsa()
6961 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerDEBUGTRAP()
6963 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
6989 SDNode *Mov = DAG.getMachineNode(AMDGPU::S_MOV_B64, DL, MVT::i64, in getSegmentAperture()
6990 DAG.getRegister(ApertureRegNo, MVT::i64)); in getSegmentAperture()
6992 ISD::TRUNCATE, DL, MVT::i32, in getSegmentAperture()
6993 DAG.getNode(ISD::SRL, DL, MVT::i64, in getSegmentAperture()
6994 {SDValue(Mov, 0), DAG.getConstant(32, DL, MVT::i64)})); in getSegmentAperture()
7003 return loadImplicitKernelArgument(DAG, MVT::i32, DL, Align(4), Param); in getSegmentAperture()
7012 return DAG.getUNDEF(MVT::i32); in getSegmentAperture()
7016 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); in getSegmentAperture()
7029 return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, in getSegmentAperture()
7075 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
7081 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
7087 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
7088 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
7090 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr, in lowerADDRSPACECAST()
7102 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
7103 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); in lowerADDRSPACECAST()
7109 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
7112 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
7114 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr, in lowerADDRSPACECAST()
7120 Op.getValueType() == MVT::i64) { in lowerADDRSPACECAST()
7123 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32); in lowerADDRSPACECAST()
7124 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi); in lowerADDRSPACECAST()
7125 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in lowerADDRSPACECAST()
7129 Src.getValueType() == MVT::i64) in lowerADDRSPACECAST()
7130 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
7165 EVT::getVectorVT(*DAG.getContext(), MVT::i32, VecNumElts / 2); in lowerINSERT_SUBVECTOR()
7166 EVT NewInsVT = InsNumElts == 2 ? MVT::i32 in lowerINSERT_SUBVECTOR()
7168 MVT::i32, InsNumElts / 2); in lowerINSERT_SUBVECTOR()
7178 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins, in lowerINSERT_SUBVECTOR()
7179 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7182 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7190 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7192 DAG.getConstant(IdxVal + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7212 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
7214 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
7215 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
7216 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
7217 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
7219 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
7220 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
7224 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
7226 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
7227 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
7229 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
7232 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
7233 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
7248 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerINSERT_VECTOR_ELT()
7253 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
7254 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
7302 SDValue V2 = DAG.getBitcast(MVT::v2i64, Vec); in lowerEXTRACT_VECTOR_ELT()
7304 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7305 DAG.getConstant(0, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
7307 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7308 DAG.getConstant(1, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
7310 SDValue V2 = DAG.getBitcast(MVT::v4i64, Vec); in lowerEXTRACT_VECTOR_ELT()
7313 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7314 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
7317 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
7319 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
7324 SDValue V2 = DAG.getBitcast(MVT::v8i64, Vec); in lowerEXTRACT_VECTOR_ELT()
7327 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7328 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
7332 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
7335 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
7350 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerEXTRACT_VECTOR_ELT()
7363 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
7366 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
7371 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) { in lowerEXTRACT_VECTOR_ELT()
7372 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
7390 EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16; in lowerVECTOR_SHUFFLE()
7412 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7424 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7428 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7457 if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v8i16 || in lowerBUILD_VECTOR()
7458 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in lowerBUILD_VECTOR()
7459 EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), in lowerBUILD_VECTOR()
7461 MVT HalfIntVT = MVT::getIntegerVT(HalfVT.getSizeInBits()); in lowerBUILD_VECTOR()
7476 SDValue Blend = DAG.getBuildVector(MVT::getVectorVT(HalfIntVT, 2), SL, in lowerBUILD_VECTOR()
7481 if (VT == MVT::v16i16 || VT == MVT::v16f16 || VT == MVT::v16bf16) { in lowerBUILD_VECTOR()
7482 EVT QuarterVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), in lowerBUILD_VECTOR()
7484 MVT QuarterIntVT = MVT::getIntegerVT(QuarterVT.getSizeInBits()); in lowerBUILD_VECTOR()
7498 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 4), SL, Casts); in lowerBUILD_VECTOR()
7502 if (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v32bf16) { in lowerBUILD_VECTOR()
7503 EVT QuarterVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), in lowerBUILD_VECTOR()
7505 MVT QuarterIntVT = MVT::getIntegerVT(QuarterVT.getSizeInBits()); in lowerBUILD_VECTOR()
7519 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 8), SL, Casts); in lowerBUILD_VECTOR()
7523 assert(VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16); in lowerBUILD_VECTOR()
7531 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7532 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7536 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
7537 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
7539 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
7540 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
7544 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7545 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7547 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
7599 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset, GAFlags); in buildPCRelGlobalAddress()
7602 PtrHi = DAG.getTargetConstant(0, DL, MVT::i32); in buildPCRelGlobalAddress()
7604 PtrHi = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset, GAFlags + 1); in buildPCRelGlobalAddress()
7629 assert(PtrVT == MVT::i32 && "32-bit pointer is expected."); in LowerGlobalAddress()
7642 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(), in LowerGlobalAddress()
7644 return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA); in LowerGlobalAddress()
7649 GV, DL, MVT::i32, GSD->getOffset(), SIInstrInfo::MO_ABS32_LO); in LowerGlobalAddress()
7650 AddrLo = {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, AddrLo), 0}; in LowerGlobalAddress()
7653 GV, DL, MVT::i32, GSD->getOffset(), SIInstrInfo::MO_ABS32_HI); in LowerGlobalAddress()
7654 AddrHi = {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, AddrHi), 0}; in LowerGlobalAddress()
7656 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, AddrLo, AddrHi); in LowerGlobalAddress()
7692 SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue, in copyToM0()
7699 MVT VT, in lowerImplicitZextParam()
7703 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); in lowerImplicitZextParam()
7705 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
7730 MVT Type; in getBuildDwordsVector()
7734 Type = MVT::getVectorVT(MVT::f32, NumElts); in getBuildDwordsVector()
7737 Type = MVT::v16f32; in getBuildDwordsVector()
7744 if (Elt.getValueType() != MVT::f32) in getBuildDwordsVector()
7745 Elt = DAG.getBitcast(MVT::f32, Elt); in getBuildDwordsVector()
7749 VecElts[i] = DAG.getUNDEF(MVT::f32); in getBuildDwordsVector()
7791 MVT DataDwordVT = NumDataDwords == 1 ? in constructRetValue()
7792 MVT::i32 : MVT::getVectorVT(MVT::i32, NumDataDwords); in constructRetValue()
7794 MVT MaskPopVT = MaskPopDwords == 1 ? in constructRetValue()
7795 MVT::i32 : MVT::getVectorVT(MVT::i32, MaskPopDwords); in constructRetValue()
7801 SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32); in constructRetValue()
7837 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, SDValue(Result, 0), in constructRetValue()
7838 DAG.getConstant(MaskPopDwords, DL, MVT::i32)); in constructRetValue()
7859 *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); in parseTexFail()
7861 *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); in parseTexFail()
7868 MVT PackVectorVT, in packImage16bitOpsToDwords()
7884 if (Addr.getValueType() != MVT::i16) in packImage16bitOpsToDwords()
7885 Addr = DAG.getBitcast(MVT::i16, Addr); in packImage16bitOpsToDwords()
7886 Addr = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Addr); in packImage16bitOpsToDwords()
7891 Addr = DAG.getBitcast(MVT::f32, Addr); in packImage16bitOpsToDwords()
7936 VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL, in lowerImage()
7939 VData = DAG.getBitcast(MVT::v4i32, VData); in lowerImage()
7941 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32; in lowerImage()
7955 MVT StoreVT = VData.getSimpleValueType(); in lowerImage()
7956 if (StoreVT.getScalarType() == MVT::f16) { in lowerImage()
7968 MVT LoadVT = ResultTypes[0].getSimpleVT(); in lowerImage()
7969 if (LoadVT.getScalarType() == MVT::f16) { in lowerImage()
7998 MVT VAddrVT = in lowerImage()
8000 MVT VAddrScalarVT = VAddrVT.getScalarType(); in lowerImage()
8001 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
8002 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; in lowerImage()
8006 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
8007 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; in lowerImage()
8011 if (IsA16 && (Op.getOperand(ArgOffset + I).getValueType() == MVT::f16)) { in lowerImage()
8016 MVT::v2f16, DL, in lowerImage()
8017 {Op.getOperand(ArgOffset + I), DAG.getUNDEF(MVT::f16)}); in lowerImage()
8110 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1); in lowerImage()
8111 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1); in lowerImage()
8154 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumVDataDwords) in lowerImage()
8155 : MVT::i32; in lowerImage()
8187 Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32)); in lowerImage()
8189 Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32)); in lowerImage()
8192 Ops.push_back(DAG.getTargetConstant(CPol, DL, MVT::i32)); in lowerImage()
8287 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) { in lowerSBuffer()
8290 DAG.getVTList(MVT::i32), Ops, VT, MMO); in lowerSBuffer()
8316 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerSBuffer()
8321 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in lowerSBuffer()
8323 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) { in lowerSBuffer()
8330 MVT LoadVT = VT.getSimpleVT(); in lowerSBuffer()
8332 assert((LoadVT.getScalarType() == MVT::i32 || in lowerSBuffer()
8333 LoadVT.getScalarType() == MVT::f32)); in lowerSBuffer()
8337 LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4); in lowerSBuffer()
8340 SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); in lowerSBuffer()
8349 Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32); in lowerSBuffer()
8365 MVT VT = MVT::i32; in lowerWaveID()
8378 return DAG.getConstant(0, SL, MVT::i32); in lowerWorkitemID()
8380 SDValue Val = loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in lowerWorkitemID()
8392 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val, in lowerWorkitemID()
8518 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
8524 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
8530 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
8557 SDLoc(Op), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8623 if (Op.getOperand(1).getValueType() == MVT::i1 && in LowerINTRINSIC_WO_CHAIN()
8675 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
8695 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0, in LowerINTRINSIC_WO_CHAIN()
8697 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; in LowerINTRINSIC_WO_CHAIN()
8705 SDValue SrcVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, in LowerINTRINSIC_WO_CHAIN()
8708 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec, in LowerINTRINSIC_WO_CHAIN()
8709 DAG.getConstant(1, SL, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
8710 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ); in LowerINTRINSIC_WO_CHAIN()
8713 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
8721 SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0, in LowerINTRINSIC_WO_CHAIN()
8723 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; in LowerINTRINSIC_WO_CHAIN()
8733 if (Op.getOperand(4).getValueType() == MVT::i32) in LowerINTRINSIC_WO_CHAIN()
8737 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8745 if (Op.getOperand(6).getValueType() == MVT::i32) in LowerINTRINSIC_WO_CHAIN()
8749 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8778 return DAG.getRegister(AMDGPU::SGPR_NULL, MVT::i32); in selectSOffset()
8795 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerRawBufferAtomicIntrin()
8800 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in lowerRawBufferAtomicIntrin()
8828 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in lowerStructBufferAtomicIntrin()
8891 DAG.getTargetConstant(Offset, DL, MVT::i16), in LowerINTRINSIC_W_CHAIN()
8914 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
8919 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8944 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8960 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
8966 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8969 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
8993 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8996 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
9115 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
9120 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
9143 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
9160 assert(NodePtr.getValueType() == MVT::i32 || in LowerINTRINSIC_W_CHAIN()
9161 NodePtr.getValueType() == MVT::i64); in LowerINTRINSIC_W_CHAIN()
9162 assert(RayDir.getValueType() == MVT::v3f16 || in LowerINTRINSIC_W_CHAIN()
9163 RayDir.getValueType() == MVT::v3f32); in LowerINTRINSIC_W_CHAIN()
9173 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16; in LowerINTRINSIC_W_CHAIN()
9174 const bool Is64 = NodePtr.getValueType() == MVT::i64; in LowerINTRINSIC_W_CHAIN()
9208 Ops.push_back(DAG.getBitcast(MVT::i32, Lanes[I])); in LowerINTRINSIC_W_CHAIN()
9212 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
9213 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
9219 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
9220 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
9223 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
9224 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
9232 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent)); in LowerINTRINSIC_W_CHAIN()
9240 MVT::i32, DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
9243 Ops.push_back(DAG.getBuildVector(MVT::v3i32, DL, MergedLanes)); in LowerINTRINSIC_W_CHAIN()
9250 DAG.ExtractVectorElements(DAG.getBitcast(MVT::v2i32, NodePtr), Ops, 0, in LowerINTRINSIC_W_CHAIN()
9255 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent)); in LowerINTRINSIC_W_CHAIN()
9264 SDValue Undef = DAG.getUNDEF(MVT::i32); in LowerINTRINSIC_W_CHAIN()
9269 MVT::getVectorVT(MVT::i32, Ops.size()), DL, Ops); in LowerINTRINSIC_W_CHAIN()
9275 Ops.push_back(DAG.getTargetConstant(IsA16, DL, MVT::i1)); in LowerINTRINSIC_W_CHAIN()
9333 SDValue K = DAG.getTargetConstant(BarID, DL, MVT::i32); in LowerINTRINSIC_W_CHAIN()
9370 EVT OpDWordsVT = EVT::getVectorVT(C, MVT::i32, NumOpDWords); in getMemIntrinsicNode()
9376 SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, in getMemIntrinsicNode()
9381 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, ZeroIdx) in getMemIntrinsicNode()
9383 EVT::getVectorVT(C, MVT::i32, NumValueDWords), Op, in getMemIntrinsicNode()
9390 (VT == MVT::v3i32 || VT == MVT::v3f32)) { in getMemIntrinsicNode()
9422 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElements); in handleD16VData()
9443 DAG.getBuildVector(MVT::v2i16, DL, {Elts[I * 2], Elts[I * 2 + 1]}); in handleD16VData()
9444 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); in handleD16VData()
9450 SDValue Pair = DAG.getBuildVector(MVT::v2i16, DL, in handleD16VData()
9451 {Elts[I * 2], DAG.getUNDEF(MVT::i16)}); in handleD16VData()
9452 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); in handleD16VData()
9457 PackedElts.resize(Elts.size(), DAG.getUNDEF(MVT::i32)); in handleD16VData()
9461 EVT::getVectorVT(*DAG.getContext(), MVT::i32, PackedElts.size()); in handleD16VData()
9504 SDValue Undef = DAG.getUNDEF(MVT::f32); in LowerINTRINSIC_VOID()
9507 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), // src0 in LowerINTRINSIC_VOID()
9508 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src1), // src1 in LowerINTRINSIC_VOID()
9512 DAG.getTargetConstant(1, DL, MVT::i1), // compr in LowerINTRINSIC_VOID()
9525 return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other, in LowerINTRINSIC_VOID()
9532 DAG.getTargetConstant(AMDGPU::Barrier::WORKGROUP, DL, MVT::i32); in LowerINTRINSIC_VOID()
9535 MVT::Other, K, Op.getOperand(0)), in LowerINTRINSIC_VOID()
9538 SDValue(DAG.getMachineNode(AMDGPU::S_BARRIER_WAIT, DL, MVT::Other, K, in LowerINTRINSIC_VOID()
9550 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
9566 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9578 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
9588 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
9594 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9633 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
9638 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9689 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9746 Ops.push_back(DAG.getBuildVector(MVT::v2i32, DL, in LowerINTRINSIC_VOID()
9760 DAG.getTargetConstant(Aux & AMDGPU::CPol::ALL, DL, MVT::i8)); // cpol in LowerINTRINSIC_VOID()
9762 Aux & AMDGPU::CPol::SWZ_pregfx12 ? 1 : 0, DL, MVT::i8)); // swz in LowerINTRINSIC_VOID()
9827 RHS.getOperand(0).getValueType() == MVT::i32) { in LowerINTRINSIC_VOID()
9839 DAG.getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, in LowerINTRINSIC_VOID()
9840 DAG.getTargetConstant(0, DL, MVT::i32)), 0); in LowerINTRINSIC_VOID()
9872 return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, in LowerINTRINSIC_VOID()
9904 SDValue K = DAG.getTargetConstant(BarVal, DL, MVT::i32); in LowerINTRINSIC_VOID()
9925 M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, Op.getOperand(3), in LowerINTRINSIC_VOID()
9926 DAG.getShiftAmountConstant(16, MVT::i32, DL)); in LowerINTRINSIC_VOID()
9932 M0Val = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, in LowerINTRINSIC_VOID()
9990 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32)); in splitBufferOffsets()
9992 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32); in splitBufferOffsets()
9997 N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops); in splitBufferOffsets()
10002 N0 = DAG.getConstant(0, DL, MVT::i32); in splitBufferOffsets()
10004 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32)); in splitBufferOffsets()
10020 Offsets[0] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
10021 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
10022 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
10034 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
10035 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
10041 ? DAG.getRegister(AMDGPU::SGPR_NULL, MVT::i32) in setBufferOffsets()
10042 : DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
10046 Offsets[2] = DAG.getTargetConstant(0, DL, MVT::i32); in setBufferOffsets()
10056 SDValue Rsrc = DAG.getBitcast(MVT::v4i32, MaybePointer); in bufferRsrcPtrToVector()
10071 auto [LowHalf, HighHalf] = DAG.SplitScalar(Pointer, Loc, MVT::i32, MVT::i32); in lowerPointerAsRsrcIntrin()
10072 SDValue Mask = DAG.getConstant(0x0000ffff, Loc, MVT::i32); in lowerPointerAsRsrcIntrin()
10073 SDValue Masked = DAG.getNode(ISD::AND, Loc, MVT::i32, HighHalf, Mask); in lowerPointerAsRsrcIntrin()
10082 ShiftedStride = DAG.getConstant(*ConstStride << 16, Loc, MVT::i32); in lowerPointerAsRsrcIntrin()
10084 SDValue ExtStride = DAG.getAnyExtOrTrunc(Stride, Loc, MVT::i32); in lowerPointerAsRsrcIntrin()
10086 DAG.getNode(ISD::SHL, Loc, MVT::i32, ExtStride, in lowerPointerAsRsrcIntrin()
10087 DAG.getShiftAmountConstant(16, MVT::i32, Loc)); in lowerPointerAsRsrcIntrin()
10089 NewHighHalf = DAG.getNode(ISD::OR, Loc, MVT::i32, Masked, ShiftedStride); in lowerPointerAsRsrcIntrin()
10092 SDValue Rsrc = DAG.getNode(ISD::BUILD_VECTOR, Loc, MVT::v4i32, LowHalf, in lowerPointerAsRsrcIntrin()
10094 SDValue RsrcPtr = DAG.getNode(ISD::BITCAST, Loc, MVT::i128, Rsrc); in lowerPointerAsRsrcIntrin()
10107 unsigned Opc = (LoadVT.getScalarType() == MVT::i8) in handleByteShortBufferLoads()
10112 SDVTList VTs = DAG.getVTList(MVT::v2i32, MVT::Other); in handleByteShortBufferLoads()
10113 SDValue Op = getMemIntrinsicNode(Opc, DL, VTs, Ops, MVT::v2i32, OpMMO, DAG); in handleByteShortBufferLoads()
10114 SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, in handleByteShortBufferLoads()
10115 DAG.getConstant(1, DL, MVT::i32)); in handleByteShortBufferLoads()
10116 SDValue Data = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, in handleByteShortBufferLoads()
10117 DAG.getConstant(0, DL, MVT::i32)); in handleByteShortBufferLoads()
10123 unsigned Opc = (LoadVT.getScalarType() == MVT::i8) ? in handleByteShortBufferLoads()
10126 SDVTList ResList = DAG.getVTList(MVT::i32, MVT::Other); in handleByteShortBufferLoads()
10140 if (VDataType == MVT::f16 || VDataType == MVT::bf16) in handleByteShortBufferStores()
10141 Ops[1] = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Ops[1]); in handleByteShortBufferStores()
10143 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]); in handleByteShortBufferStores()
10145 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE : in handleByteShortBufferStores()
10202 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
10203 Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(), in widenLoad()
10216 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
10256 if (MemVT == MVT::i16 && isTypeLegal(MVT::i16)) in LowerLOAD()
10266 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16; in LowerLOAD()
10268 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, in LowerLOAD()
10282 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD, in LowerLOAD()
10283 DAG.getConstant(I, DL, MVT::i32)); in LowerLOAD()
10285 Elts.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Elt)); in LowerLOAD()
10299 assert(Op.getValueType().getVectorElementType() == MVT::i32 && in LowerLOAD()
10422 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerSELECT()
10423 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerSELECT()
10425 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1)); in LowerSELECT()
10426 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2)); in LowerSELECT()
10428 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero); in LowerSELECT()
10429 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero); in LowerSELECT()
10431 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1); in LowerSELECT()
10433 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One); in LowerSELECT()
10434 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One); in LowerSELECT()
10436 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1); in LowerSELECT()
10438 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi}); in LowerSELECT()
10459 if (!AllowInaccurateRcp && VT != MVT::f16) in lowerFastUnsafeFDIV()
10488 if (!AllowInaccurateRcp && (VT != MVT::f16 || !Flags.hasAllowReciprocal())) in lowerFastUnsafeFDIV()
10533 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPBinOp()
10555 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPTernOp()
10576 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
10577 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
10579 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
10580 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
10582 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
10583 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
10585 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
10595 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags); in lowerFDIV_FAST()
10598 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
10601 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
10603 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
10606 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); in lowerFDIV_FAST()
10610 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags); in lowerFDIV_FAST()
10612 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags); in lowerFDIV_FAST()
10615 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags); in lowerFDIV_FAST()
10617 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags); in lowerFDIV_FAST()
10619 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags); in lowerFDIV_FAST()
10630 return DAG.getTargetConstant(Mode, SDLoc(), MVT::i32); in getSPDenormModeValue()
10648 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
10650 SDVTList ScaleVT = DAG.getVTList(MVT::f32, MVT::i1); in LowerFDIV32()
10658 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
10660 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
10665 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32); in LowerFDIV32()
10683 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerFDIV32()
10688 DAG.getVTList(MVT::i32, MVT::Glue), in LowerFDIV32()
10706 SL, MVT::i32); in LowerFDIV32()
10720 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
10723 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
10726 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
10729 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
10732 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, in LowerFDIV32()
10735 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
10744 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, in LowerFDIV32()
10752 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
10755 AMDGPU::S_SETREG_B32, SL, MVT::Other, in LowerFDIV32()
10759 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
10765 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
10768 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags); in LowerFDIV32()
10779 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
10781 SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1); in LowerFDIV64()
10785 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
10787 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
10789 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
10791 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
10793 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
10797 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
10798 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
10800 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
10809 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
10812 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
10813 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
10814 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
10815 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
10817 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
10818 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
10821 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
10823 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
10825 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
10826 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
10827 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
10832 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
10835 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
10841 if (VT == MVT::f32) in LowerFDIV()
10844 if (VT == MVT::f64) in LowerFDIV()
10847 if (VT == MVT::f16) in LowerFDIV()
10858 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32; in LowerFFREXP()
10862 DAG.getTargetConstant(Intrinsic::amdgcn_frexp_mant, dl, MVT::i32), Val); in LowerFFREXP()
10866 DAG.getTargetConstant(Intrinsic::amdgcn_frexp_exp, dl, MVT::i32), Val); in LowerFFREXP()
10873 SDValue IsFinite = DAG.getSetCC(dl, MVT::i1, Fabs, Inf, ISD::SETOLT); in LowerFFREXP()
10888 if (VT == MVT::i1) { in LowerSTORE()
10890 DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32), in LowerSTORE()
10891 Store->getBasePtr(), MVT::i1, Store->getMemOperand()); in LowerSTORE()
10895 Store->getValue().getValueType().getScalarType() == MVT::i32); in LowerSTORE()
10968 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags); in lowerFSQRTF16()
10970 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32); in lowerFSQRTF16()
10972 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags); in lowerFSQRTF16()
10974 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt, in lowerFSQRTF16()
10975 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFSQRTF16()
10981 MVT VT = Op.getValueType().getSimpleVT(); in lowerFSQRTF32()
10988 DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, DL, MVT::i32), X, Flags); in lowerFSQRTF32()
10992 SDValue NeedScale = DAG.getSetCC(DL, MVT::i1, X, ScaleThreshold, ISD::SETOLT); in lowerFSQRTF32()
11004 DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, DL, MVT::i32); in lowerFSQRTF32()
11007 SDValue SqrtSAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, SqrtS); in lowerFSQRTF32()
11008 SDValue SqrtSNextDownInt = DAG.getNode(ISD::ADD, DL, MVT::i32, SqrtSAsInt, in lowerFSQRTF32()
11009 DAG.getConstant(-1, DL, MVT::i32)); in lowerFSQRTF32()
11018 SDValue SqrtSNextUpInt = DAG.getNode(ISD::ADD, DL, MVT::i32, SqrtSAsInt, in lowerFSQRTF32()
11019 DAG.getConstant(1, DL, MVT::i32)); in lowerFSQRTF32()
11027 SDValue SqrtVPLE0 = DAG.getSetCC(DL, MVT::i1, SqrtVP, Zero, ISD::SETOLE); in lowerFSQRTF32()
11032 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, MVT::i1, SqrtVS, Zero, ISD::SETOGT); in lowerFSQRTF32()
11061 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, SqrtX, in lowerFSQRTF32()
11062 DAG.getTargetConstant(fcZero | fcPosInf, DL, MVT::i32)); in lowerFSQRTF32()
11093 SDValue ScaleConstant = DAG.getConstantFP(0x1.0p-767, DL, MVT::f64); in lowerFSQRTF64()
11095 SDValue Scaling = DAG.getSetCC(DL, MVT::i1, X, ScaleConstant, ISD::SETOLT); in lowerFSQRTF64()
11097 SDValue ZeroInt = DAG.getConstant(0, DL, MVT::i32); in lowerFSQRTF64()
11100 SDValue ScaleUpFactor = DAG.getConstant(256, DL, MVT::i32); in lowerFSQRTF64()
11102 DAG.getNode(ISD::SELECT, DL, MVT::i32, Scaling, ScaleUpFactor, ZeroInt); in lowerFSQRTF64()
11103 SDValue SqrtX = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, X, ScaleUp, Flags); in lowerFSQRTF64()
11105 SDValue SqrtY = DAG.getNode(AMDGPUISD::RSQ, DL, MVT::f64, SqrtX); in lowerFSQRTF64()
11107 SDValue SqrtS0 = DAG.getNode(ISD::FMUL, DL, MVT::f64, SqrtX, SqrtY); in lowerFSQRTF64()
11109 SDValue Half = DAG.getConstantFP(0.5, DL, MVT::f64); in lowerFSQRTF64()
11110 SDValue SqrtH0 = DAG.getNode(ISD::FMUL, DL, MVT::f64, SqrtY, Half); in lowerFSQRTF64()
11112 SDValue NegSqrtH0 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtH0); in lowerFSQRTF64()
11113 SDValue SqrtR0 = DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtH0, SqrtS0, Half); in lowerFSQRTF64()
11115 SDValue SqrtH1 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtH0, SqrtR0, SqrtH0); in lowerFSQRTF64()
11117 SDValue SqrtS1 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtS0, SqrtR0, SqrtS0); in lowerFSQRTF64()
11119 SDValue NegSqrtS1 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtS1); in lowerFSQRTF64()
11120 SDValue SqrtD0 = DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtS1, SqrtS1, SqrtX); in lowerFSQRTF64()
11122 SDValue SqrtS2 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtD0, SqrtH1, SqrtS1); in lowerFSQRTF64()
11124 SDValue NegSqrtS2 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtS2); in lowerFSQRTF64()
11126 DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtS2, SqrtS2, SqrtX); in lowerFSQRTF64()
11128 SDValue SqrtRet = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtD1, SqrtH1, SqrtS2); in lowerFSQRTF64()
11130 SDValue ScaleDownFactor = DAG.getConstant(-128, DL, MVT::i32); in lowerFSQRTF64()
11132 DAG.getNode(ISD::SELECT, DL, MVT::i32, Scaling, ScaleDownFactor, ZeroInt); in lowerFSQRTF64()
11133 SqrtRet = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, SqrtRet, ScaleDown, Flags); in lowerFSQRTF64()
11140 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, SqrtX, in lowerFSQRTF64()
11141 DAG.getTargetConstant(fcZero | fcPosInf, DL, MVT::i32)); in lowerFSQRTF64()
11144 return DAG.getNode(ISD::SELECT, DL, MVT::f64, IsZeroOrInf, SqrtX, SqrtRet, in lowerFSQRTF64()
11194 MVT SimpleVT = VT.getSimpleVT(); in LowerATOMIC_CMP_SWAP()
11195 MVT VecType = MVT::getVectorVT(SimpleVT, 2); in LowerATOMIC_CMP_SWAP()
11212 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16) in performUCharToFloatCombine()
11225 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) { in performUCharToFloatCombine()
11227 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src); in performUCharToFloatCombine()
11231 if (ScalarVT != MVT::f32) { in performUCharToFloatCombine()
11233 DAG.getTargetConstant(0, DL, MVT::i32)); in performUCharToFloatCombine()
11252 if (MagnitudeOp.getValueType() == MVT::f64) { in performFCopySignCombine()
11253 SDValue MagAsVector = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, MagnitudeOp); in performFCopySignCombine()
11255 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector, in performFCopySignCombine()
11256 DAG.getConstant(0, DL, MVT::i32)); in performFCopySignCombine()
11258 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector, in performFCopySignCombine()
11259 DAG.getConstant(1, DL, MVT::i32)); in performFCopySignCombine()
11262 DAG.getNode(ISD::FCOPYSIGN, DL, MVT::f32, MagHi, SignOp); in performFCopySignCombine()
11264 SDValue Vector = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, MagLo, HiOp); in performFCopySignCombine()
11266 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, Vector); in performFCopySignCombine()
11269 if (SignOp.getValueType() != MVT::f64) in performFCopySignCombine()
11277 SDValue SignAsVector = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, SignOp); in performFCopySignCombine()
11279 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, SignAsVector, in performFCopySignCombine()
11280 DAG.getConstant(1, DL, MVT::i32)); in performFCopySignCombine()
11425 if (V.getValueType() != MVT::i1) in isBoolSGPR()
11516 if (VT == MVT::i64 && CRHS) { in performAndCombine()
11522 if (CRHS && VT == MVT::i32) { in performAndCombine()
11537 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
11539 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
11540 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
11545 DAG.getConstant(NB, SDLoc(CRHS), MVT::i32)); in performAndCombine()
11561 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performAndCombine()
11562 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
11601 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performAndCombine()
11602 X, DAG.getConstant(Mask, DL, MVT::i32)); in performAndCombine()
11625 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, RHS.getOperand(0), in performAndCombine()
11626 DAG.getConstant(NewMask, DL, MVT::i32)); in performAndCombine()
11630 if (VT == MVT::i32 && in performAndCombine()
11636 return DAG.getSelect(SDLoc(N), MVT::i32, RHS.getOperand(0), in performAndCombine()
11637 LHS, DAG.getConstant(0, SDLoc(N), MVT::i32)); in performAndCombine()
11642 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performAndCombine()
11681 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performAndCombine()
11683 DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
12137 return DAG.getBitcastedAnyExtOrTrunc(Src, SL, MVT::i32); in getDWordFromOffset()
12143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Src, in getDWordFromOffset()
12144 DAG.getConstant(DWordOffset, SL, MVT::i32)); in getDWordFromOffset()
12149 DAG.getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32)); in getDWordFromOffset()
12153 DAG.getConstant(ShiftVal, SL, MVT::i32)); in getDWordFromOffset()
12154 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32); in getDWordFromOffset()
12171 MVT::getVectorVT(MVT::getIntegerVT(ScalarTySize), NumAvailElements), SL, in getDWordFromOffset()
12173 return Ret = DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32); in getDWordFromOffset()
12179 DAG.getConstant(ShiftVal, SL, MVT::i32)); in getDWordFromOffset()
12180 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32); in getDWordFromOffset()
12189 assert(VT == MVT::i32); in matchPERM()
12245 return DAG.getBitcast(MVT::getIntegerVT(32), Op); in matchPERM()
12265 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, MVT::i32); in matchPERM()
12266 OtherOp = DAG.getBitcastedAnyExtOrTrunc(OtherOp, DL, MVT::i32); in matchPERM()
12268 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op, OtherOp, in matchPERM()
12269 DAG.getConstant(PermMask, DL, MVT::i32)); in matchPERM()
12281 if (VT == MVT::i1) { in performOrCombine()
12299 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performOrCombine()
12300 Src, DAG.getConstant(NewMask, DL, MVT::i32)); in performOrCombine()
12316 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performOrCombine()
12317 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
12322 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performOrCombine()
12381 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performOrCombine()
12383 DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
12392 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps()) in performOrCombine()
12407 if (SrcVT == MVT::i32) { in performOrCombine()
12411 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
12416 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
12418 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
12445 if (CRHS && VT == MVT::i64) { in performXorCombine()
12453 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) { in performXorCombine()
12461 DAG.getNode(ISD::BITCAST, DL, MVT::f32, LHS->getOperand(1)); in performXorCombine()
12463 DAG.getNode(ISD::BITCAST, DL, MVT::f32, LHS->getOperand(2)); in performXorCombine()
12464 SDValue FNegLHS = DAG.getNode(ISD::FNEG, DL, MVT::f32, CastLHS); in performXorCombine()
12465 SDValue FNegRHS = DAG.getNode(ISD::FNEG, DL, MVT::f32, CastRHS); in performXorCombine()
12466 SDValue NewSelect = DAG.getNode(ISD::SELECT, DL, MVT::f32, in performXorCombine()
12482 if (VT != MVT::i32) in performZeroExtendCombine()
12486 if (Src.getValueType() != MVT::i16) in performZeroExtendCombine()
12501 VTSign->getVT() == MVT::i8) || in performSignExtendInRegCombine()
12503 VTSign->getVT() == MVT::i16))) { in performSignExtendInRegCombine()
12512 SDVTList ResList = DCI.DAG.getVTList(MVT::i32); in performSignExtendInRegCombine()
12525 VTSign->getVT() == MVT::i8) || in performSignExtendInRegCombine()
12527 VTSign->getVT() == MVT::i16)) && in performSignExtendInRegCombine()
12541 SDVTList ResList = DCI.DAG.getVTList(MVT::i32, in performSignExtendInRegCombine()
12562 return DAG.getConstant(0, SDLoc(N), MVT::i1); in performClassCombine()
12565 return DAG.getUNDEF(MVT::i1); in performClassCombine()
12581 if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP || in performRcpCombine()
12588 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) && in performRcpCombine()
12670 if (Op.getValueType() == MVT::i32) { in isCanonicalized()
12686 return Op.getValueType().getScalarType() != MVT::f16; in isCanonicalized()
12754 if (Op.getValueType() == MVT::i16) { in isCanonicalized()
12756 if (TruncSrc.getValueType() == MVT::i32 && in isCanonicalized()
12758 TruncSrc.getOperand(0).getValueType() == MVT::v2f16) { in isCanonicalized()
12978 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 && in performFCanonicalizeCombine()
12979 isTypeLegal(MVT::v2f16)) { in performFCanonicalizeCombine()
13074 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16())) in performIntMed3ImmCombine()
13126 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) { in performFPMed3ImmCombine()
13158 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()); in supportsMin3Max3()
13161 return (VT == MVT::f32 || VT == MVT::f16) && Subtarget.hasIEEEMinMax3(); in supportsMin3Max3()
13166 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16()); in supportsMin3Max3()
13242 (VT == MVT::f32 || VT == MVT::f64 || in performMinMaxCombine()
13243 (VT == MVT::f16 && Subtarget->has16BitInsts()) || in performMinMaxCombine()
13244 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) && in performMinMaxCombine()
13468 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
13469 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13471 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
13472 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13523 Src.getOperand(0).getValueType() == MVT::f16) { in strictFPExtFromF16()
13532 return DAG.getConstantFP(Val, SDLoc(Src), MVT::f16); in strictFPExtFromF16()
13545 if (VT != MVT::f16) in performFPRoundCombine()
13549 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse()) in performFPRoundCombine()
13590 if (((VT == MVT::f32 && in getFusedOpcode()
13592 (VT == MVT::f16 && Subtarget->hasMadF16() && in getFusedOpcode()
13613 if (VT != MVT::i32 && VT != MVT::i64) in reassociateScalarOps()
13650 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i1); in getMad64_32()
13729 if (VT != MVT::i64) { in tryFoldToMad64_32()
13730 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS); in tryFoldToMad64_32()
13731 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS); in tryFoldToMad64_32()
13732 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS); in tryFoldToMad64_32()
13747 SDValue One = DAG.getConstant(1, SL, MVT::i32); in tryFoldToMad64_32()
13749 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS); in tryFoldToMad64_32()
13750 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS); in tryFoldToMad64_32()
13752 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo); in tryFoldToMad64_32()
13756 std::tie(AccumLo, AccumHi) = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32); in tryFoldToMad64_32()
13760 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One); in tryFoldToMad64_32()
13761 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo); in tryFoldToMad64_32()
13762 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13767 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One); in tryFoldToMad64_32()
13768 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi); in tryFoldToMad64_32()
13769 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13772 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi}); in tryFoldToMad64_32()
13773 Accum = DAG.getBitcast(MVT::i64, Accum); in tryFoldToMad64_32()
13776 if (VT != MVT::i64) in tryFoldToMad64_32()
13907 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp, in resolveSources()
13908 DAG.getConstant(Elt->PermMask, SL, MVT::i32)); in resolveSources()
13934 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal, in resolveSources()
13936 DAG.getConstant(PermMask, SL, MVT::i32))); in resolveSources()
13950 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp, in resolveSources()
13951 DAG.getConstant(FirstElt->PermMask, SL, MVT::i32))); in resolveSources()
13958 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1]) in resolveSources()
14114 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32)); in performAddCombine()
14175 MVT::getIntegerVT(32)); in performAddCombine()
14177 MVT::getIntegerVT(32)); in performAddCombine()
14188 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32); in performAddCombine()
14192 SL, MVT::i64); in performAddCombine()
14195 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0, in performAddCombine()
14196 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1)); in performAddCombine()
14201 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG()) in performAddCombine()
14222 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performAddCombine()
14223 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
14243 if (VT != MVT::i32) in performSubCombine()
14263 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performSubCombine()
14264 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performSubCombine()
14283 if (N->getValueType(0) != MVT::i32) in performAddCarrySubCarryCombine()
14398 if (VT != MVT::f16 || !Subtarget->has16BitInsts()) in performFDivCombine()
14434 if (!Subtarget->hasDot7Insts() || VT != MVT::f32) in performFMACombine()
14491 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16) in performFMACombine()
14496 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
14497 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
14523 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND && in performSetCCCombine()
14533 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14534 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14559 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14560 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14567 if (VT != MVT::f32 && VT != MVT::f64 && in performSetCCCombine()
14568 (!Subtarget->has16BitInsts() || VT != MVT::f16)) in performSetCCCombine()
14591 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14592 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
14620 SDLoc(Shift.getOperand(0)), MVT::i32); in performCvtF32UByteNCombine()
14630 MVT::f32, Shifted); in performCvtF32UByteNCombine()
14648 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc); in performCvtF32UByteNCombine()
14716 if (N->getValueType(0) == MVT::i32 && N->isDivergent() && in PerformDAGCombine()
14767 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) { in PerformDAGCombine()
14771 if (EltVT != MVT::i16) in PerformDAGCombine()
14772 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
14774 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
14926 Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32)); in adjustWritemask()
14929 MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); in adjustWritemask()
14931 MVT ResultVT = NewChannels == 1 ? in adjustWritemask()
14932 SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 : in adjustWritemask()
14935 DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT); in adjustWritemask()
14965 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); in adjustWritemask()
15004 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) { in legalizeTargetIndependentNode()
15008 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1); in legalizeTargetIndependentNode()
15073 MVT VT = Src0.getValueType().getSimpleVT(); in PostISelFolding()
15283 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32); in buildSMovImm32()
15284 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0); in buildSMovImm32()
15296 DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
15298 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in wrapAddr64Rsrc()
15300 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32) in wrapAddr64Rsrc()
15304 MVT::v2i32, Ops0), 0); in wrapAddr64Rsrc()
15308 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
15310 DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32), in wrapAddr64Rsrc()
15312 DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32) in wrapAddr64Rsrc()
15315 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1); in wrapAddr64Rsrc()
15325 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr); in buildRSRC()
15326 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr); in buildRSRC()
15328 PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi, in buildRSRC()
15329 DAG.getConstant(RsrcDword1, DL, MVT::i32)), in buildRSRC()
15338 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), in buildRSRC()
15340 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in buildRSRC()
15342 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32), in buildRSRC()
15344 DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32), in buildRSRC()
15346 DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32) in buildRSRC()
15349 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops); in buildRSRC()
15359 MVT VT) const { in getRegForInlineAsmConstraint()
15413 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 || in getRegForInlineAsmConstraint()
15414 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16)) in getRegForInlineAsmConstraint()
15516 Ops.push_back(DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64)); in LowerAsmOperandForConstraint()
15594 MVT VT = Op.getSimpleValueType(); in checkAsmConstraintValA()
15598 case MVT::i16: in checkAsmConstraintValA()
15600 case MVT::f16: in checkAsmConstraintValA()
15602 case MVT::bf16: in checkAsmConstraintValA()
15604 case MVT::v2i16: in checkAsmConstraintValA()
15606 case MVT::v2f16: in checkAsmConstraintValA()
15608 case MVT::v2bf16: in checkAsmConstraintValA()
16021 case MVT::f32: in denormalsEnabledForType()
16023 case MVT::f64: in denormalsEnabledForType()
16024 case MVT::f16: in denormalsEnabledForType()
16343 SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const { in getRegClassFor()
16490 if (User->getOperand(Op)->getValueType(ResNo) != MVT::i1) in checkForPhysRegDependency()