1 /* Internal functions.
2 Copyright (C) 2011-2022 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "predict.h"
29 #include "stringpool.h"
30 #include "tree-vrp.h"
31 #include "tree-ssanames.h"
32 #include "expmed.h"
33 #include "memmodel.h"
34 #include "optabs.h"
35 #include "emit-rtl.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
40 #include "dojump.h"
41 #include "expr.h"
42 #include "stringpool.h"
43 #include "attribs.h"
44 #include "asan.h"
45 #include "ubsan.h"
46 #include "recog.h"
47 #include "builtins.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
52 #include "explow.h"
53 #include "rtl-iter.h"
54 #include "gimple-range.h"
55 #include "tree-ssa-live.h"
56 #include "tree-outof-ssa.h"
57
58 /* For lang_hooks.types.type_for_mode. */
59 #include "langhooks.h"
60
61 /* The names of each internal function, indexed by function number. */
62 const char *const internal_fn_name_array[] = {
63 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
64 #include "internal-fn.def"
65 "<invalid-fn>"
66 };
67
68 /* The ECF_* flags of each internal function, indexed by function number. */
69 const int internal_fn_flags_array[] = {
70 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
71 #include "internal-fn.def"
72 0
73 };
74
75 /* Return the internal function called NAME, or IFN_LAST if there's
76 no such function. */
77
78 internal_fn
lookup_internal_fn(const char * name)79 lookup_internal_fn (const char *name)
80 {
81 typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
82 static name_to_fn_map_type *name_to_fn_map;
83
84 if (!name_to_fn_map)
85 {
86 name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
87 for (unsigned int i = 0; i < IFN_LAST; ++i)
88 name_to_fn_map->put (internal_fn_name (internal_fn (i)),
89 internal_fn (i));
90 }
91 internal_fn *entry = name_to_fn_map->get (name);
92 return entry ? *entry : IFN_LAST;
93 }
94
95 /* Fnspec of each internal function, indexed by function number. */
96 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
97
98 void
init_internal_fns()99 init_internal_fns ()
100 {
101 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
102 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
103 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
104 #include "internal-fn.def"
105 internal_fn_fnspec_array[IFN_LAST] = 0;
106 }
107
108 /* Create static initializers for the information returned by
109 direct_internal_fn. */
110 #define not_direct { -2, -2, false }
111 #define mask_load_direct { -1, 2, false }
112 #define load_lanes_direct { -1, -1, false }
113 #define mask_load_lanes_direct { -1, -1, false }
114 #define gather_load_direct { 3, 1, false }
115 #define len_load_direct { -1, -1, false }
116 #define mask_store_direct { 3, 2, false }
117 #define store_lanes_direct { 0, 0, false }
118 #define mask_store_lanes_direct { 0, 0, false }
119 #define vec_cond_mask_direct { 1, 0, false }
120 #define vec_cond_direct { 2, 0, false }
121 #define scatter_store_direct { 3, 1, false }
122 #define len_store_direct { 3, 3, false }
123 #define vec_set_direct { 3, 3, false }
124 #define unary_direct { 0, 0, true }
125 #define binary_direct { 0, 0, true }
126 #define ternary_direct { 0, 0, true }
127 #define cond_unary_direct { 1, 1, true }
128 #define cond_binary_direct { 1, 1, true }
129 #define cond_ternary_direct { 1, 1, true }
130 #define while_direct { 0, 2, false }
131 #define fold_extract_direct { 2, 2, false }
132 #define fold_left_direct { 1, 1, false }
133 #define mask_fold_left_direct { 1, 1, false }
134 #define check_ptrs_direct { 0, 0, false }
135
136 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
137 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
138 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
139 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
140 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
141 #include "internal-fn.def"
142 not_direct
143 };
144
145 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
146 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
147
148 static enum insn_code
get_multi_vector_move(tree array_type,convert_optab optab)149 get_multi_vector_move (tree array_type, convert_optab optab)
150 {
151 machine_mode imode;
152 machine_mode vmode;
153
154 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
155 imode = TYPE_MODE (array_type);
156 vmode = TYPE_MODE (TREE_TYPE (array_type));
157
158 return convert_optab_handler (optab, imode, vmode);
159 }
160
161 /* Expand LOAD_LANES call STMT using optab OPTAB. */
162
163 static void
expand_load_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)164 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
165 {
166 class expand_operand ops[2];
167 tree type, lhs, rhs;
168 rtx target, mem;
169
170 lhs = gimple_call_lhs (stmt);
171 rhs = gimple_call_arg (stmt, 0);
172 type = TREE_TYPE (lhs);
173
174 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
175 mem = expand_normal (rhs);
176
177 gcc_assert (MEM_P (mem));
178 PUT_MODE (mem, TYPE_MODE (type));
179
180 create_output_operand (&ops[0], target, TYPE_MODE (type));
181 create_fixed_operand (&ops[1], mem);
182 expand_insn (get_multi_vector_move (type, optab), 2, ops);
183 if (!rtx_equal_p (target, ops[0].value))
184 emit_move_insn (target, ops[0].value);
185 }
186
187 /* Expand STORE_LANES call STMT using optab OPTAB. */
188
189 static void
expand_store_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)190 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
191 {
192 class expand_operand ops[2];
193 tree type, lhs, rhs;
194 rtx target, reg;
195
196 lhs = gimple_call_lhs (stmt);
197 rhs = gimple_call_arg (stmt, 0);
198 type = TREE_TYPE (rhs);
199
200 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
201 reg = expand_normal (rhs);
202
203 gcc_assert (MEM_P (target));
204 PUT_MODE (target, TYPE_MODE (type));
205
206 create_fixed_operand (&ops[0], target);
207 create_input_operand (&ops[1], reg, TYPE_MODE (type));
208 expand_insn (get_multi_vector_move (type, optab), 2, ops);
209 }
210
211 static void
expand_ANNOTATE(internal_fn,gcall *)212 expand_ANNOTATE (internal_fn, gcall *)
213 {
214 gcc_unreachable ();
215 }
216
217 /* This should get expanded in omp_device_lower pass. */
218
219 static void
expand_GOMP_USE_SIMT(internal_fn,gcall *)220 expand_GOMP_USE_SIMT (internal_fn, gcall *)
221 {
222 gcc_unreachable ();
223 }
224
225 /* This should get expanded in omp_device_lower pass. */
226
227 static void
expand_GOMP_SIMT_ENTER(internal_fn,gcall *)228 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
229 {
230 gcc_unreachable ();
231 }
232
233 /* Allocate per-lane storage and begin non-uniform execution region. */
234
235 static void
expand_GOMP_SIMT_ENTER_ALLOC(internal_fn,gcall * stmt)236 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
237 {
238 rtx target;
239 tree lhs = gimple_call_lhs (stmt);
240 if (lhs)
241 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
242 else
243 target = gen_reg_rtx (Pmode);
244 rtx size = expand_normal (gimple_call_arg (stmt, 0));
245 rtx align = expand_normal (gimple_call_arg (stmt, 1));
246 class expand_operand ops[3];
247 create_output_operand (&ops[0], target, Pmode);
248 create_input_operand (&ops[1], size, Pmode);
249 create_input_operand (&ops[2], align, Pmode);
250 gcc_assert (targetm.have_omp_simt_enter ());
251 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
252 if (!rtx_equal_p (target, ops[0].value))
253 emit_move_insn (target, ops[0].value);
254 }
255
256 /* Deallocate per-lane storage and leave non-uniform execution region. */
257
258 static void
expand_GOMP_SIMT_EXIT(internal_fn,gcall * stmt)259 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
260 {
261 gcc_checking_assert (!gimple_call_lhs (stmt));
262 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
263 class expand_operand ops[1];
264 create_input_operand (&ops[0], arg, Pmode);
265 gcc_assert (targetm.have_omp_simt_exit ());
266 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
267 }
268
269 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
270 without SIMT execution this should be expanded in omp_device_lower pass. */
271
272 static void
expand_GOMP_SIMT_LANE(internal_fn,gcall * stmt)273 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
274 {
275 tree lhs = gimple_call_lhs (stmt);
276 if (!lhs)
277 return;
278
279 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
280 gcc_assert (targetm.have_omp_simt_lane ());
281 emit_insn (targetm.gen_omp_simt_lane (target));
282 }
283
284 /* This should get expanded in omp_device_lower pass. */
285
286 static void
expand_GOMP_SIMT_VF(internal_fn,gcall *)287 expand_GOMP_SIMT_VF (internal_fn, gcall *)
288 {
289 gcc_unreachable ();
290 }
291
292 /* Lane index of the first SIMT lane that supplies a non-zero argument.
293 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
294 lane that executed the last iteration for handling OpenMP lastprivate. */
295
296 static void
expand_GOMP_SIMT_LAST_LANE(internal_fn,gcall * stmt)297 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
298 {
299 tree lhs = gimple_call_lhs (stmt);
300 if (!lhs)
301 return;
302
303 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
304 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
305 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
306 class expand_operand ops[2];
307 create_output_operand (&ops[0], target, mode);
308 create_input_operand (&ops[1], cond, mode);
309 gcc_assert (targetm.have_omp_simt_last_lane ());
310 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
311 if (!rtx_equal_p (target, ops[0].value))
312 emit_move_insn (target, ops[0].value);
313 }
314
315 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
316
317 static void
expand_GOMP_SIMT_ORDERED_PRED(internal_fn,gcall * stmt)318 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
319 {
320 tree lhs = gimple_call_lhs (stmt);
321 if (!lhs)
322 return;
323
324 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
325 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
326 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
327 class expand_operand ops[2];
328 create_output_operand (&ops[0], target, mode);
329 create_input_operand (&ops[1], ctr, mode);
330 gcc_assert (targetm.have_omp_simt_ordered ());
331 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
332 if (!rtx_equal_p (target, ops[0].value))
333 emit_move_insn (target, ops[0].value);
334 }
335
336 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
337 any lane supplies a non-zero argument. */
338
339 static void
expand_GOMP_SIMT_VOTE_ANY(internal_fn,gcall * stmt)340 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
341 {
342 tree lhs = gimple_call_lhs (stmt);
343 if (!lhs)
344 return;
345
346 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
347 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
348 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
349 class expand_operand ops[2];
350 create_output_operand (&ops[0], target, mode);
351 create_input_operand (&ops[1], cond, mode);
352 gcc_assert (targetm.have_omp_simt_vote_any ());
353 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
354 if (!rtx_equal_p (target, ops[0].value))
355 emit_move_insn (target, ops[0].value);
356 }
357
358 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
359 is destination lane index XOR given offset. */
360
361 static void
expand_GOMP_SIMT_XCHG_BFLY(internal_fn,gcall * stmt)362 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
363 {
364 tree lhs = gimple_call_lhs (stmt);
365 if (!lhs)
366 return;
367
368 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
369 rtx src = expand_normal (gimple_call_arg (stmt, 0));
370 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
371 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
372 class expand_operand ops[3];
373 create_output_operand (&ops[0], target, mode);
374 create_input_operand (&ops[1], src, mode);
375 create_input_operand (&ops[2], idx, SImode);
376 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
377 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
378 if (!rtx_equal_p (target, ops[0].value))
379 emit_move_insn (target, ops[0].value);
380 }
381
382 /* Exchange between SIMT lanes according to given source lane index. */
383
384 static void
expand_GOMP_SIMT_XCHG_IDX(internal_fn,gcall * stmt)385 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
386 {
387 tree lhs = gimple_call_lhs (stmt);
388 if (!lhs)
389 return;
390
391 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
392 rtx src = expand_normal (gimple_call_arg (stmt, 0));
393 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
394 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
395 class expand_operand ops[3];
396 create_output_operand (&ops[0], target, mode);
397 create_input_operand (&ops[1], src, mode);
398 create_input_operand (&ops[2], idx, SImode);
399 gcc_assert (targetm.have_omp_simt_xchg_idx ());
400 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
401 if (!rtx_equal_p (target, ops[0].value))
402 emit_move_insn (target, ops[0].value);
403 }
404
405 /* This should get expanded in adjust_simduid_builtins. */
406
407 static void
expand_GOMP_SIMD_LANE(internal_fn,gcall *)408 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
409 {
410 gcc_unreachable ();
411 }
412
413 /* This should get expanded in adjust_simduid_builtins. */
414
415 static void
expand_GOMP_SIMD_VF(internal_fn,gcall *)416 expand_GOMP_SIMD_VF (internal_fn, gcall *)
417 {
418 gcc_unreachable ();
419 }
420
421 /* This should get expanded in adjust_simduid_builtins. */
422
423 static void
expand_GOMP_SIMD_LAST_LANE(internal_fn,gcall *)424 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
425 {
426 gcc_unreachable ();
427 }
428
429 /* This should get expanded in adjust_simduid_builtins. */
430
431 static void
expand_GOMP_SIMD_ORDERED_START(internal_fn,gcall *)432 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
433 {
434 gcc_unreachable ();
435 }
436
437 /* This should get expanded in adjust_simduid_builtins. */
438
439 static void
expand_GOMP_SIMD_ORDERED_END(internal_fn,gcall *)440 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
441 {
442 gcc_unreachable ();
443 }
444
445 /* This should get expanded in the sanopt pass. */
446
447 static void
expand_UBSAN_NULL(internal_fn,gcall *)448 expand_UBSAN_NULL (internal_fn, gcall *)
449 {
450 gcc_unreachable ();
451 }
452
453 /* This should get expanded in the sanopt pass. */
454
455 static void
expand_UBSAN_BOUNDS(internal_fn,gcall *)456 expand_UBSAN_BOUNDS (internal_fn, gcall *)
457 {
458 gcc_unreachable ();
459 }
460
461 /* This should get expanded in the sanopt pass. */
462
463 static void
expand_UBSAN_VPTR(internal_fn,gcall *)464 expand_UBSAN_VPTR (internal_fn, gcall *)
465 {
466 gcc_unreachable ();
467 }
468
469 /* This should get expanded in the sanopt pass. */
470
471 static void
expand_UBSAN_PTR(internal_fn,gcall *)472 expand_UBSAN_PTR (internal_fn, gcall *)
473 {
474 gcc_unreachable ();
475 }
476
477 /* This should get expanded in the sanopt pass. */
478
479 static void
expand_UBSAN_OBJECT_SIZE(internal_fn,gcall *)480 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
481 {
482 gcc_unreachable ();
483 }
484
485 /* This should get expanded in the sanopt pass. */
486
487 static void
expand_HWASAN_CHECK(internal_fn,gcall *)488 expand_HWASAN_CHECK (internal_fn, gcall *)
489 {
490 gcc_unreachable ();
491 }
492
493 /* For hwasan stack tagging:
494 Clear tags on the dynamically allocated space.
495 For use after an object dynamically allocated on the stack goes out of
496 scope. */
497 static void
expand_HWASAN_ALLOCA_UNPOISON(internal_fn,gcall * gc)498 expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
499 {
500 gcc_assert (Pmode == ptr_mode);
501 tree restored_position = gimple_call_arg (gc, 0);
502 rtx restored_rtx = expand_expr (restored_position, NULL_RTX, VOIDmode,
503 EXPAND_NORMAL);
504 rtx func = init_one_libfunc ("__hwasan_tag_memory");
505 rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
506 stack_pointer_rtx, NULL_RTX, 0,
507 OPTAB_WIDEN);
508 emit_library_call_value (func, NULL_RTX, LCT_NORMAL, VOIDmode,
509 virtual_stack_dynamic_rtx, Pmode,
510 HWASAN_STACK_BACKGROUND, QImode,
511 off, Pmode);
512 }
513
514 /* For hwasan stack tagging:
515 Return a tag to be used for a dynamic allocation. */
516 static void
expand_HWASAN_CHOOSE_TAG(internal_fn,gcall * gc)517 expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
518 {
519 tree tag = gimple_call_lhs (gc);
520 rtx target = expand_expr (tag, NULL_RTX, VOIDmode, EXPAND_NORMAL);
521 machine_mode mode = GET_MODE (target);
522 gcc_assert (mode == QImode);
523
524 rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
525 gcc_assert (base_tag);
526 rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
527 rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
528 target, /* unsignedp = */1,
529 OPTAB_WIDEN);
530 chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
531
532 /* Really need to put the tag into the `target` RTX. */
533 if (chosen_tag != target)
534 {
535 rtx temp = chosen_tag;
536 gcc_assert (GET_MODE (chosen_tag) == mode);
537 emit_move_insn (target, temp);
538 }
539
540 hwasan_increment_frame_tag ();
541 }
542
543 /* For hwasan stack tagging:
544 Tag a region of space in the shadow stack according to the base pointer of
545 an object on the stack. N.b. the length provided in the internal call is
546 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
547 static void
expand_HWASAN_MARK(internal_fn,gcall * gc)548 expand_HWASAN_MARK (internal_fn, gcall *gc)
549 {
550 gcc_assert (ptr_mode == Pmode);
551 HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gc, 0));
552 bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
553
554 tree base = gimple_call_arg (gc, 1);
555 gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
556 rtx base_rtx = expand_normal (base);
557
558 rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
559 : targetm.memtag.extract_tag (base_rtx, NULL_RTX);
560 rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
561
562 tree len = gimple_call_arg (gc, 2);
563 rtx r_len = expand_normal (len);
564
565 rtx func = init_one_libfunc ("__hwasan_tag_memory");
566 emit_library_call (func, LCT_NORMAL, VOIDmode, address, Pmode,
567 tag, QImode, r_len, Pmode);
568 }
569
570 /* For hwasan stack tagging:
571 Store a tag into a pointer. */
572 static void
expand_HWASAN_SET_TAG(internal_fn,gcall * gc)573 expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
574 {
575 gcc_assert (ptr_mode == Pmode);
576 tree g_target = gimple_call_lhs (gc);
577 tree g_ptr = gimple_call_arg (gc, 0);
578 tree g_tag = gimple_call_arg (gc, 1);
579
580 rtx ptr = expand_normal (g_ptr);
581 rtx tag = expand_expr (g_tag, NULL_RTX, QImode, EXPAND_NORMAL);
582 rtx target = expand_normal (g_target);
583
584 rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
585 rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
586 if (tagged_value != target)
587 emit_move_insn (target, tagged_value);
588 }
589
590 /* This should get expanded in the sanopt pass. */
591
592 static void
expand_ASAN_CHECK(internal_fn,gcall *)593 expand_ASAN_CHECK (internal_fn, gcall *)
594 {
595 gcc_unreachable ();
596 }
597
598 /* This should get expanded in the sanopt pass. */
599
600 static void
expand_ASAN_MARK(internal_fn,gcall *)601 expand_ASAN_MARK (internal_fn, gcall *)
602 {
603 gcc_unreachable ();
604 }
605
606 /* This should get expanded in the sanopt pass. */
607
608 static void
expand_ASAN_POISON(internal_fn,gcall *)609 expand_ASAN_POISON (internal_fn, gcall *)
610 {
611 gcc_unreachable ();
612 }
613
614 /* This should get expanded in the sanopt pass. */
615
616 static void
expand_ASAN_POISON_USE(internal_fn,gcall *)617 expand_ASAN_POISON_USE (internal_fn, gcall *)
618 {
619 gcc_unreachable ();
620 }
621
622 /* This should get expanded in the tsan pass. */
623
624 static void
expand_TSAN_FUNC_EXIT(internal_fn,gcall *)625 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
626 {
627 gcc_unreachable ();
628 }
629
630 /* This should get expanded in the lower pass. */
631
632 static void
expand_FALLTHROUGH(internal_fn,gcall * call)633 expand_FALLTHROUGH (internal_fn, gcall *call)
634 {
635 error_at (gimple_location (call),
636 "invalid use of attribute %<fallthrough%>");
637 }
638
639 /* Return minimum precision needed to represent all values
640 of ARG in SIGNed integral type. */
641
642 static int
get_min_precision(tree arg,signop sign)643 get_min_precision (tree arg, signop sign)
644 {
645 int prec = TYPE_PRECISION (TREE_TYPE (arg));
646 int cnt = 0;
647 signop orig_sign = sign;
648 if (TREE_CODE (arg) == INTEGER_CST)
649 {
650 int p;
651 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
652 {
653 widest_int w = wi::to_widest (arg);
654 w = wi::ext (w, prec, sign);
655 p = wi::min_precision (w, sign);
656 }
657 else
658 p = wi::min_precision (wi::to_wide (arg), sign);
659 return MIN (p, prec);
660 }
661 while (CONVERT_EXPR_P (arg)
662 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
663 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
664 {
665 arg = TREE_OPERAND (arg, 0);
666 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
667 {
668 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
669 sign = UNSIGNED;
670 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
671 return prec + (orig_sign != sign);
672 prec = TYPE_PRECISION (TREE_TYPE (arg));
673 }
674 if (++cnt > 30)
675 return prec + (orig_sign != sign);
676 }
677 if (CONVERT_EXPR_P (arg)
678 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
679 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
680 {
681 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
682 If y_2's min precision is smaller than prec, return that. */
683 int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
684 if (oprec < prec)
685 return oprec + (orig_sign != sign);
686 }
687 if (TREE_CODE (arg) != SSA_NAME)
688 return prec + (orig_sign != sign);
689 value_range r;
690 while (!get_global_range_query ()->range_of_expr (r, arg)
691 || r.kind () != VR_RANGE)
692 {
693 gimple *g = SSA_NAME_DEF_STMT (arg);
694 if (is_gimple_assign (g)
695 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
696 {
697 tree t = gimple_assign_rhs1 (g);
698 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
699 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
700 {
701 arg = t;
702 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
703 {
704 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
705 sign = UNSIGNED;
706 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
707 return prec + (orig_sign != sign);
708 prec = TYPE_PRECISION (TREE_TYPE (arg));
709 }
710 if (++cnt > 30)
711 return prec + (orig_sign != sign);
712 continue;
713 }
714 }
715 return prec + (orig_sign != sign);
716 }
717 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
718 {
719 int p1 = wi::min_precision (r.lower_bound (), sign);
720 int p2 = wi::min_precision (r.upper_bound (), sign);
721 p1 = MAX (p1, p2);
722 prec = MIN (prec, p1);
723 }
724 else if (sign == UNSIGNED && !wi::neg_p (r.lower_bound (), SIGNED))
725 {
726 int p = wi::min_precision (r.upper_bound (), UNSIGNED);
727 prec = MIN (prec, p);
728 }
729 return prec + (orig_sign != sign);
730 }
731
732 /* Helper for expand_*_overflow. Set the __imag__ part to true
733 (1 except for signed:1 type, in which case store -1). */
734
735 static void
expand_arith_set_overflow(tree lhs,rtx target)736 expand_arith_set_overflow (tree lhs, rtx target)
737 {
738 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
739 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
740 write_complex_part (target, constm1_rtx, true);
741 else
742 write_complex_part (target, const1_rtx, true);
743 }
744
745 /* Helper for expand_*_overflow. Store RES into the __real__ part
746 of TARGET. If RES has larger MODE than __real__ part of TARGET,
747 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
748 if LHS has smaller precision than its mode. */
749
750 static void
expand_arith_overflow_result_store(tree lhs,rtx target,scalar_int_mode mode,rtx res)751 expand_arith_overflow_result_store (tree lhs, rtx target,
752 scalar_int_mode mode, rtx res)
753 {
754 scalar_int_mode tgtmode
755 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
756 rtx lres = res;
757 if (tgtmode != mode)
758 {
759 rtx_code_label *done_label = gen_label_rtx ();
760 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
761 lres = convert_modes (tgtmode, mode, res, uns);
762 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
763 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
764 EQ, true, mode, NULL_RTX, NULL, done_label,
765 profile_probability::very_likely ());
766 expand_arith_set_overflow (lhs, target);
767 emit_label (done_label);
768 }
769 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
770 int tgtprec = GET_MODE_PRECISION (tgtmode);
771 if (prec < tgtprec)
772 {
773 rtx_code_label *done_label = gen_label_rtx ();
774 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
775 res = lres;
776 if (uns)
777 {
778 rtx mask
779 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
780 tgtmode);
781 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
782 true, OPTAB_LIB_WIDEN);
783 }
784 else
785 {
786 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
787 NULL_RTX, 1);
788 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
789 NULL_RTX, 0);
790 }
791 do_compare_rtx_and_jump (res, lres,
792 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
793 profile_probability::very_likely ());
794 expand_arith_set_overflow (lhs, target);
795 emit_label (done_label);
796 }
797 write_complex_part (target, lres, false);
798 }
799
800 /* Helper for expand_*_overflow. Store RES into TARGET. */
801
802 static void
expand_ubsan_result_store(rtx target,rtx res)803 expand_ubsan_result_store (rtx target, rtx res)
804 {
805 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
806 /* If this is a scalar in a register that is stored in a wider mode
807 than the declared mode, compute the result into its declared mode
808 and then convert to the wider mode. Our value is the computed
809 expression. */
810 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
811 else
812 emit_move_insn (target, res);
813 }
814
815 /* Add sub/add overflow checking to the statement STMT.
816 CODE says whether the operation is +, or -. */
817
818 void
expand_addsub_overflow(location_t loc,tree_code code,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan,tree * datap)819 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
820 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
821 bool uns1_p, bool is_ubsan, tree *datap)
822 {
823 rtx res, target = NULL_RTX;
824 tree fn;
825 rtx_code_label *done_label = gen_label_rtx ();
826 rtx_code_label *do_error = gen_label_rtx ();
827 do_pending_stack_adjust ();
828 rtx op0 = expand_normal (arg0);
829 rtx op1 = expand_normal (arg1);
830 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
831 int prec = GET_MODE_PRECISION (mode);
832 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
833 bool do_xor = false;
834
835 if (is_ubsan)
836 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
837
838 if (lhs)
839 {
840 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
841 if (!is_ubsan)
842 write_complex_part (target, const0_rtx, true);
843 }
844
845 /* We assume both operands and result have the same precision
846 here (GET_MODE_BITSIZE (mode)), S stands for signed type
847 with that precision, U for unsigned type with that precision,
848 sgn for unsigned most significant bit in that precision.
849 s1 is signed first operand, u1 is unsigned first operand,
850 s2 is signed second operand, u2 is unsigned second operand,
851 sr is signed result, ur is unsigned result and the following
852 rules say how to compute result (which is always result of
853 the operands as if both were unsigned, cast to the right
854 signedness) and how to compute whether operation overflowed.
855
856 s1 + s2 -> sr
857 res = (S) ((U) s1 + (U) s2)
858 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
859 s1 - s2 -> sr
860 res = (S) ((U) s1 - (U) s2)
861 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
862 u1 + u2 -> ur
863 res = u1 + u2
864 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
865 u1 - u2 -> ur
866 res = u1 - u2
867 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
868 s1 + u2 -> sr
869 res = (S) ((U) s1 + u2)
870 ovf = ((U) res ^ sgn) < u2
871 s1 + u2 -> ur
872 t1 = (S) (u2 ^ sgn)
873 t2 = s1 + t1
874 res = (U) t2 ^ sgn
875 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
876 s1 - u2 -> sr
877 res = (S) ((U) s1 - u2)
878 ovf = u2 > ((U) s1 ^ sgn)
879 s1 - u2 -> ur
880 res = (U) s1 - u2
881 ovf = s1 < 0 || u2 > (U) s1
882 u1 - s2 -> sr
883 res = u1 - (U) s2
884 ovf = u1 >= ((U) s2 ^ sgn)
885 u1 - s2 -> ur
886 t1 = u1 ^ sgn
887 t2 = t1 - (U) s2
888 res = t2 ^ sgn
889 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
890 s1 + s2 -> ur
891 res = (U) s1 + (U) s2
892 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
893 u1 + u2 -> sr
894 res = (S) (u1 + u2)
895 ovf = (U) res < u2 || res < 0
896 u1 - u2 -> sr
897 res = (S) (u1 - u2)
898 ovf = u1 >= u2 ? res < 0 : res >= 0
899 s1 - s2 -> ur
900 res = (U) s1 - (U) s2
901 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
902
903 if (code == PLUS_EXPR && uns0_p && !uns1_p)
904 {
905 /* PLUS_EXPR is commutative, if operand signedness differs,
906 canonicalize to the first operand being signed and second
907 unsigned to simplify following code. */
908 std::swap (op0, op1);
909 std::swap (arg0, arg1);
910 uns0_p = false;
911 uns1_p = true;
912 }
913
914 /* u1 +- u2 -> ur */
915 if (uns0_p && uns1_p && unsr_p)
916 {
917 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
918 : usubv4_optab, mode);
919 if (icode != CODE_FOR_nothing)
920 {
921 class expand_operand ops[4];
922 rtx_insn *last = get_last_insn ();
923
924 res = gen_reg_rtx (mode);
925 create_output_operand (&ops[0], res, mode);
926 create_input_operand (&ops[1], op0, mode);
927 create_input_operand (&ops[2], op1, mode);
928 create_fixed_operand (&ops[3], do_error);
929 if (maybe_expand_insn (icode, 4, ops))
930 {
931 last = get_last_insn ();
932 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
933 && JUMP_P (last)
934 && any_condjump_p (last)
935 && !find_reg_note (last, REG_BR_PROB, 0))
936 add_reg_br_prob_note (last,
937 profile_probability::very_unlikely ());
938 emit_jump (done_label);
939 goto do_error_label;
940 }
941
942 delete_insns_since (last);
943 }
944
945 /* Compute the operation. On RTL level, the addition is always
946 unsigned. */
947 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
948 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
949 rtx tem = op0;
950 /* For PLUS_EXPR, the operation is commutative, so we can pick
951 operand to compare against. For prec <= BITS_PER_WORD, I think
952 preferring REG operand is better over CONST_INT, because
953 the CONST_INT might enlarge the instruction or CSE would need
954 to figure out we'd already loaded it into a register before.
955 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
956 as then the multi-word comparison can be perhaps simplified. */
957 if (code == PLUS_EXPR
958 && (prec <= BITS_PER_WORD
959 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
960 : CONST_SCALAR_INT_P (op1)))
961 tem = op1;
962 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
963 true, mode, NULL_RTX, NULL, done_label,
964 profile_probability::very_likely ());
965 goto do_error_label;
966 }
967
968 /* s1 +- u2 -> sr */
969 if (!uns0_p && uns1_p && !unsr_p)
970 {
971 /* Compute the operation. On RTL level, the addition is always
972 unsigned. */
973 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
974 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
975 rtx tem = expand_binop (mode, add_optab,
976 code == PLUS_EXPR ? res : op0, sgn,
977 NULL_RTX, false, OPTAB_LIB_WIDEN);
978 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
979 done_label, profile_probability::very_likely ());
980 goto do_error_label;
981 }
982
983 /* s1 + u2 -> ur */
984 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
985 {
986 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
987 OPTAB_LIB_WIDEN);
988 /* As we've changed op1, we have to avoid using the value range
989 for the original argument. */
990 arg1 = error_mark_node;
991 do_xor = true;
992 goto do_signed;
993 }
994
995 /* u1 - s2 -> ur */
996 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
997 {
998 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
999 OPTAB_LIB_WIDEN);
1000 /* As we've changed op0, we have to avoid using the value range
1001 for the original argument. */
1002 arg0 = error_mark_node;
1003 do_xor = true;
1004 goto do_signed;
1005 }
1006
1007 /* s1 - u2 -> ur */
1008 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1009 {
1010 /* Compute the operation. On RTL level, the addition is always
1011 unsigned. */
1012 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1013 OPTAB_LIB_WIDEN);
1014 int pos_neg = get_range_pos_neg (arg0);
1015 if (pos_neg == 2)
1016 /* If ARG0 is known to be always negative, this is always overflow. */
1017 emit_jump (do_error);
1018 else if (pos_neg == 3)
1019 /* If ARG0 is not known to be always positive, check at runtime. */
1020 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1021 NULL, do_error, profile_probability::very_unlikely ());
1022 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1023 done_label, profile_probability::very_likely ());
1024 goto do_error_label;
1025 }
1026
1027 /* u1 - s2 -> sr */
1028 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1029 {
1030 /* Compute the operation. On RTL level, the addition is always
1031 unsigned. */
1032 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1033 OPTAB_LIB_WIDEN);
1034 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1035 OPTAB_LIB_WIDEN);
1036 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1037 done_label, profile_probability::very_likely ());
1038 goto do_error_label;
1039 }
1040
1041 /* u1 + u2 -> sr */
1042 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1043 {
1044 /* Compute the operation. On RTL level, the addition is always
1045 unsigned. */
1046 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1047 OPTAB_LIB_WIDEN);
1048 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1049 NULL, do_error, profile_probability::very_unlikely ());
1050 rtx tem = op1;
1051 /* The operation is commutative, so we can pick operand to compare
1052 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1053 is better over CONST_INT, because the CONST_INT might enlarge the
1054 instruction or CSE would need to figure out we'd already loaded it
1055 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1056 might be more beneficial, as then the multi-word comparison can be
1057 perhaps simplified. */
1058 if (prec <= BITS_PER_WORD
1059 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1060 : CONST_SCALAR_INT_P (op0))
1061 tem = op0;
1062 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1063 done_label, profile_probability::very_likely ());
1064 goto do_error_label;
1065 }
1066
1067 /* s1 +- s2 -> ur */
1068 if (!uns0_p && !uns1_p && unsr_p)
1069 {
1070 /* Compute the operation. On RTL level, the addition is always
1071 unsigned. */
1072 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1073 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1074 int pos_neg = get_range_pos_neg (arg1);
1075 if (code == PLUS_EXPR)
1076 {
1077 int pos_neg0 = get_range_pos_neg (arg0);
1078 if (pos_neg0 != 3 && pos_neg == 3)
1079 {
1080 std::swap (op0, op1);
1081 pos_neg = pos_neg0;
1082 }
1083 }
1084 rtx tem;
1085 if (pos_neg != 3)
1086 {
1087 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1088 ? and_optab : ior_optab,
1089 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1090 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1091 NULL, done_label, profile_probability::very_likely ());
1092 }
1093 else
1094 {
1095 rtx_code_label *do_ior_label = gen_label_rtx ();
1096 do_compare_rtx_and_jump (op1, const0_rtx,
1097 code == MINUS_EXPR ? GE : LT, false, mode,
1098 NULL_RTX, NULL, do_ior_label,
1099 profile_probability::even ());
1100 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1101 OPTAB_LIB_WIDEN);
1102 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1103 NULL, done_label, profile_probability::very_likely ());
1104 emit_jump (do_error);
1105 emit_label (do_ior_label);
1106 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1107 OPTAB_LIB_WIDEN);
1108 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1109 NULL, done_label, profile_probability::very_likely ());
1110 }
1111 goto do_error_label;
1112 }
1113
1114 /* u1 - u2 -> sr */
1115 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1116 {
1117 /* Compute the operation. On RTL level, the addition is always
1118 unsigned. */
1119 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1120 OPTAB_LIB_WIDEN);
1121 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1122 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1123 op0_geu_op1, profile_probability::even ());
1124 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1125 NULL, done_label, profile_probability::very_likely ());
1126 emit_jump (do_error);
1127 emit_label (op0_geu_op1);
1128 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1129 NULL, done_label, profile_probability::very_likely ());
1130 goto do_error_label;
1131 }
1132
1133 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1134
1135 /* s1 +- s2 -> sr */
1136 do_signed:
1137 {
1138 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
1139 : subv4_optab, mode);
1140 if (icode != CODE_FOR_nothing)
1141 {
1142 class expand_operand ops[4];
1143 rtx_insn *last = get_last_insn ();
1144
1145 res = gen_reg_rtx (mode);
1146 create_output_operand (&ops[0], res, mode);
1147 create_input_operand (&ops[1], op0, mode);
1148 create_input_operand (&ops[2], op1, mode);
1149 create_fixed_operand (&ops[3], do_error);
1150 if (maybe_expand_insn (icode, 4, ops))
1151 {
1152 last = get_last_insn ();
1153 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1154 && JUMP_P (last)
1155 && any_condjump_p (last)
1156 && !find_reg_note (last, REG_BR_PROB, 0))
1157 add_reg_br_prob_note (last,
1158 profile_probability::very_unlikely ());
1159 emit_jump (done_label);
1160 goto do_error_label;
1161 }
1162
1163 delete_insns_since (last);
1164 }
1165
1166 /* Compute the operation. On RTL level, the addition is always
1167 unsigned. */
1168 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1169 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1170
1171 /* If we can prove that one of the arguments (for MINUS_EXPR only
1172 the second operand, as subtraction is not commutative) is always
1173 non-negative or always negative, we can do just one comparison
1174 and conditional jump. */
1175 int pos_neg = get_range_pos_neg (arg1);
1176 if (code == PLUS_EXPR)
1177 {
1178 int pos_neg0 = get_range_pos_neg (arg0);
1179 if (pos_neg0 != 3 && pos_neg == 3)
1180 {
1181 std::swap (op0, op1);
1182 pos_neg = pos_neg0;
1183 }
1184 }
1185
1186 /* Addition overflows if and only if the two operands have the same sign,
1187 and the result has the opposite sign. Subtraction overflows if and
1188 only if the two operands have opposite sign, and the subtrahend has
1189 the same sign as the result. Here 0 is counted as positive. */
1190 if (pos_neg == 3)
1191 {
1192 /* Compute op0 ^ op1 (operands have opposite sign). */
1193 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1194 OPTAB_LIB_WIDEN);
1195
1196 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1197 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1198 OPTAB_LIB_WIDEN);
1199
1200 rtx tem;
1201 if (code == PLUS_EXPR)
1202 {
1203 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1204 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1205 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1206 OPTAB_LIB_WIDEN);
1207 }
1208 else
1209 {
1210 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1211 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1212 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1213 OPTAB_LIB_WIDEN);
1214 }
1215
1216 /* No overflow if the result has bit sign cleared. */
1217 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1218 NULL, done_label, profile_probability::very_likely ());
1219 }
1220
1221 /* Compare the result of the operation with the first operand.
1222 No overflow for addition if second operand is positive and result
1223 is larger or second operand is negative and result is smaller.
1224 Likewise for subtraction with sign of second operand flipped. */
1225 else
1226 do_compare_rtx_and_jump (res, op0,
1227 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1228 false, mode, NULL_RTX, NULL, done_label,
1229 profile_probability::very_likely ());
1230 }
1231
1232 do_error_label:
1233 emit_label (do_error);
1234 if (is_ubsan)
1235 {
1236 /* Expand the ubsan builtin call. */
1237 push_temp_slots ();
1238 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1239 arg0, arg1, datap);
1240 expand_normal (fn);
1241 pop_temp_slots ();
1242 do_pending_stack_adjust ();
1243 }
1244 else if (lhs)
1245 expand_arith_set_overflow (lhs, target);
1246
1247 /* We're done. */
1248 emit_label (done_label);
1249
1250 if (lhs)
1251 {
1252 if (is_ubsan)
1253 expand_ubsan_result_store (target, res);
1254 else
1255 {
1256 if (do_xor)
1257 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1258 OPTAB_LIB_WIDEN);
1259
1260 expand_arith_overflow_result_store (lhs, target, mode, res);
1261 }
1262 }
1263 }
1264
1265 /* Add negate overflow checking to the statement STMT. */
1266
1267 static void
expand_neg_overflow(location_t loc,tree lhs,tree arg1,bool is_ubsan,tree * datap)1268 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1269 tree *datap)
1270 {
1271 rtx res, op1;
1272 tree fn;
1273 rtx_code_label *done_label, *do_error;
1274 rtx target = NULL_RTX;
1275
1276 done_label = gen_label_rtx ();
1277 do_error = gen_label_rtx ();
1278
1279 do_pending_stack_adjust ();
1280 op1 = expand_normal (arg1);
1281
1282 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1283 if (lhs)
1284 {
1285 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1286 if (!is_ubsan)
1287 write_complex_part (target, const0_rtx, true);
1288 }
1289
1290 enum insn_code icode = optab_handler (negv3_optab, mode);
1291 if (icode != CODE_FOR_nothing)
1292 {
1293 class expand_operand ops[3];
1294 rtx_insn *last = get_last_insn ();
1295
1296 res = gen_reg_rtx (mode);
1297 create_output_operand (&ops[0], res, mode);
1298 create_input_operand (&ops[1], op1, mode);
1299 create_fixed_operand (&ops[2], do_error);
1300 if (maybe_expand_insn (icode, 3, ops))
1301 {
1302 last = get_last_insn ();
1303 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1304 && JUMP_P (last)
1305 && any_condjump_p (last)
1306 && !find_reg_note (last, REG_BR_PROB, 0))
1307 add_reg_br_prob_note (last,
1308 profile_probability::very_unlikely ());
1309 emit_jump (done_label);
1310 }
1311 else
1312 {
1313 delete_insns_since (last);
1314 icode = CODE_FOR_nothing;
1315 }
1316 }
1317
1318 if (icode == CODE_FOR_nothing)
1319 {
1320 /* Compute the operation. On RTL level, the addition is always
1321 unsigned. */
1322 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1323
1324 /* Compare the operand with the most negative value. */
1325 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1326 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1327 done_label, profile_probability::very_likely ());
1328 }
1329
1330 emit_label (do_error);
1331 if (is_ubsan)
1332 {
1333 /* Expand the ubsan builtin call. */
1334 push_temp_slots ();
1335 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1336 arg1, NULL_TREE, datap);
1337 expand_normal (fn);
1338 pop_temp_slots ();
1339 do_pending_stack_adjust ();
1340 }
1341 else if (lhs)
1342 expand_arith_set_overflow (lhs, target);
1343
1344 /* We're done. */
1345 emit_label (done_label);
1346
1347 if (lhs)
1348 {
1349 if (is_ubsan)
1350 expand_ubsan_result_store (target, res);
1351 else
1352 expand_arith_overflow_result_store (lhs, target, mode, res);
1353 }
1354 }
1355
1356 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1357 mode MODE can be expanded without using a libcall. */
1358
1359 static bool
can_widen_mult_without_libcall(scalar_int_mode wmode,scalar_int_mode mode,rtx op0,rtx op1,bool uns)1360 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1361 rtx op0, rtx op1, bool uns)
1362 {
1363 if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1364 != CODE_FOR_nothing)
1365 return true;
1366
1367 if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1368 != CODE_FOR_nothing)
1369 return true;
1370
1371 rtx_insn *last = get_last_insn ();
1372 if (CONSTANT_P (op0))
1373 op0 = convert_modes (wmode, mode, op0, uns);
1374 else
1375 op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1376 if (CONSTANT_P (op1))
1377 op1 = convert_modes (wmode, mode, op1, uns);
1378 else
1379 op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1380 rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1381 delete_insns_since (last);
1382 return ret != NULL_RTX;
1383 }
1384
1385 /* Add mul overflow checking to the statement STMT. */
1386
1387 static void
expand_mul_overflow(location_t loc,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan,tree * datap)1388 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1389 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1390 tree *datap)
1391 {
1392 rtx res, op0, op1;
1393 tree fn, type;
1394 rtx_code_label *done_label, *do_error;
1395 rtx target = NULL_RTX;
1396 signop sign;
1397 enum insn_code icode;
1398 int save_flag_trapv = flag_trapv;
1399
1400 /* We don't want any __mulv?i3 etc. calls from the expansion of
1401 these internal functions, so disable -ftrapv temporarily. */
1402 flag_trapv = 0;
1403 done_label = gen_label_rtx ();
1404 do_error = gen_label_rtx ();
1405
1406 do_pending_stack_adjust ();
1407 op0 = expand_normal (arg0);
1408 op1 = expand_normal (arg1);
1409
1410 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1411 bool uns = unsr_p;
1412 if (lhs)
1413 {
1414 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1415 if (!is_ubsan)
1416 write_complex_part (target, const0_rtx, true);
1417 }
1418
1419 if (is_ubsan)
1420 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1421
1422 /* We assume both operands and result have the same precision
1423 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1424 with that precision, U for unsigned type with that precision,
1425 sgn for unsigned most significant bit in that precision.
1426 s1 is signed first operand, u1 is unsigned first operand,
1427 s2 is signed second operand, u2 is unsigned second operand,
1428 sr is signed result, ur is unsigned result and the following
1429 rules say how to compute result (which is always result of
1430 the operands as if both were unsigned, cast to the right
1431 signedness) and how to compute whether operation overflowed.
1432 main_ovf (false) stands for jump on signed multiplication
1433 overflow or the main algorithm with uns == false.
1434 main_ovf (true) stands for jump on unsigned multiplication
1435 overflow or the main algorithm with uns == true.
1436
1437 s1 * s2 -> sr
1438 res = (S) ((U) s1 * (U) s2)
1439 ovf = main_ovf (false)
1440 u1 * u2 -> ur
1441 res = u1 * u2
1442 ovf = main_ovf (true)
1443 s1 * u2 -> ur
1444 res = (U) s1 * u2
1445 ovf = (s1 < 0 && u2) || main_ovf (true)
1446 u1 * u2 -> sr
1447 res = (S) (u1 * u2)
1448 ovf = res < 0 || main_ovf (true)
1449 s1 * u2 -> sr
1450 res = (S) ((U) s1 * u2)
1451 ovf = (S) u2 >= 0 ? main_ovf (false)
1452 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1453 s1 * s2 -> ur
1454 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1455 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1456 res = t1 * t2
1457 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1458
1459 if (uns0_p && !uns1_p)
1460 {
1461 /* Multiplication is commutative, if operand signedness differs,
1462 canonicalize to the first operand being signed and second
1463 unsigned to simplify following code. */
1464 std::swap (op0, op1);
1465 std::swap (arg0, arg1);
1466 uns0_p = false;
1467 uns1_p = true;
1468 }
1469
1470 int pos_neg0 = get_range_pos_neg (arg0);
1471 int pos_neg1 = get_range_pos_neg (arg1);
1472
1473 /* s1 * u2 -> ur */
1474 if (!uns0_p && uns1_p && unsr_p)
1475 {
1476 switch (pos_neg0)
1477 {
1478 case 1:
1479 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1480 goto do_main;
1481 case 2:
1482 /* If s1 is negative, avoid the main code, just multiply and
1483 signal overflow if op1 is not 0. */
1484 struct separate_ops ops;
1485 ops.code = MULT_EXPR;
1486 ops.type = TREE_TYPE (arg1);
1487 ops.op0 = make_tree (ops.type, op0);
1488 ops.op1 = make_tree (ops.type, op1);
1489 ops.op2 = NULL_TREE;
1490 ops.location = loc;
1491 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1492 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1493 NULL, done_label, profile_probability::very_likely ());
1494 goto do_error_label;
1495 case 3:
1496 if (get_min_precision (arg1, UNSIGNED)
1497 + get_min_precision (arg0, SIGNED) <= GET_MODE_PRECISION (mode))
1498 {
1499 /* If the first operand is sign extended from narrower type, the
1500 second operand is zero extended from narrower type and
1501 the sum of the two precisions is smaller or equal to the
1502 result precision: if the first argument is at runtime
1503 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1504 and there will be no overflow, if the first argument is
1505 negative and the second argument zero, the result will be
1506 0 and there will be no overflow, if the first argument is
1507 negative and the second argument positive, the result when
1508 treated as signed will be negative (minimum -0x7f80 or
1509 -0x7f..f80..0) there will be always overflow. So, do
1510 res = (U) (s1 * u2)
1511 ovf = (S) res < 0 */
1512 struct separate_ops ops;
1513 ops.code = MULT_EXPR;
1514 ops.type
1515 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1516 1);
1517 ops.op0 = make_tree (ops.type, op0);
1518 ops.op1 = make_tree (ops.type, op1);
1519 ops.op2 = NULL_TREE;
1520 ops.location = loc;
1521 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1522 do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1523 mode, NULL_RTX, NULL, done_label,
1524 profile_probability::very_likely ());
1525 goto do_error_label;
1526 }
1527 rtx_code_label *do_main_label;
1528 do_main_label = gen_label_rtx ();
1529 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1530 NULL, do_main_label, profile_probability::very_likely ());
1531 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1532 NULL, do_main_label, profile_probability::very_likely ());
1533 expand_arith_set_overflow (lhs, target);
1534 emit_label (do_main_label);
1535 goto do_main;
1536 default:
1537 gcc_unreachable ();
1538 }
1539 }
1540
1541 /* u1 * u2 -> sr */
1542 if (uns0_p && uns1_p && !unsr_p)
1543 {
1544 if ((pos_neg0 | pos_neg1) == 1)
1545 {
1546 /* If both arguments are zero extended from narrower types,
1547 the MSB will be clear on both and so we can pretend it is
1548 a normal s1 * s2 -> sr multiplication. */
1549 uns0_p = false;
1550 uns1_p = false;
1551 }
1552 else
1553 uns = true;
1554 /* Rest of handling of this case after res is computed. */
1555 goto do_main;
1556 }
1557
1558 /* s1 * u2 -> sr */
1559 if (!uns0_p && uns1_p && !unsr_p)
1560 {
1561 switch (pos_neg1)
1562 {
1563 case 1:
1564 goto do_main;
1565 case 2:
1566 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1567 avoid the main code, just multiply and signal overflow
1568 unless 0 * u2 or -1 * ((U) Smin). */
1569 struct separate_ops ops;
1570 ops.code = MULT_EXPR;
1571 ops.type = TREE_TYPE (arg1);
1572 ops.op0 = make_tree (ops.type, op0);
1573 ops.op1 = make_tree (ops.type, op1);
1574 ops.op2 = NULL_TREE;
1575 ops.location = loc;
1576 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1577 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1578 NULL, done_label, profile_probability::very_likely ());
1579 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1580 NULL, do_error, profile_probability::very_unlikely ());
1581 int prec;
1582 prec = GET_MODE_PRECISION (mode);
1583 rtx sgn;
1584 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1585 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1586 NULL, done_label, profile_probability::very_likely ());
1587 goto do_error_label;
1588 case 3:
1589 /* Rest of handling of this case after res is computed. */
1590 goto do_main;
1591 default:
1592 gcc_unreachable ();
1593 }
1594 }
1595
1596 /* s1 * s2 -> ur */
1597 if (!uns0_p && !uns1_p && unsr_p)
1598 {
1599 rtx tem;
1600 switch (pos_neg0 | pos_neg1)
1601 {
1602 case 1: /* Both operands known to be non-negative. */
1603 goto do_main;
1604 case 2: /* Both operands known to be negative. */
1605 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1606 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1607 /* Avoid looking at arg0/arg1 ranges, as we've changed
1608 the arguments. */
1609 arg0 = error_mark_node;
1610 arg1 = error_mark_node;
1611 goto do_main;
1612 case 3:
1613 if ((pos_neg0 ^ pos_neg1) == 3)
1614 {
1615 /* If one operand is known to be negative and the other
1616 non-negative, this overflows always, unless the non-negative
1617 one is 0. Just do normal multiply and set overflow
1618 unless one of the operands is 0. */
1619 struct separate_ops ops;
1620 ops.code = MULT_EXPR;
1621 ops.type
1622 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1623 1);
1624 ops.op0 = make_tree (ops.type, op0);
1625 ops.op1 = make_tree (ops.type, op1);
1626 ops.op2 = NULL_TREE;
1627 ops.location = loc;
1628 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1629 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1630 true, mode, NULL_RTX, NULL, done_label,
1631 profile_probability::very_likely ());
1632 goto do_error_label;
1633 }
1634 if (get_min_precision (arg0, SIGNED)
1635 + get_min_precision (arg1, SIGNED) <= GET_MODE_PRECISION (mode))
1636 {
1637 /* If both operands are sign extended from narrower types and
1638 the sum of the two precisions is smaller or equal to the
1639 result precision: if both arguments are at runtime
1640 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1641 and there will be no overflow, if both arguments are negative,
1642 maximum result will be 0x40..00 and there will be no overflow
1643 either, if one argument is positive and the other argument
1644 negative, the result when treated as signed will be negative
1645 and there will be always overflow, and if one argument is
1646 zero and the other negative the result will be zero and no
1647 overflow. So, do
1648 res = (U) (s1 * s2)
1649 ovf = (S) res < 0 */
1650 struct separate_ops ops;
1651 ops.code = MULT_EXPR;
1652 ops.type
1653 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1654 1);
1655 ops.op0 = make_tree (ops.type, op0);
1656 ops.op1 = make_tree (ops.type, op1);
1657 ops.op2 = NULL_TREE;
1658 ops.location = loc;
1659 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1660 do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1661 mode, NULL_RTX, NULL, done_label,
1662 profile_probability::very_likely ());
1663 goto do_error_label;
1664 }
1665 /* The general case, do all the needed comparisons at runtime. */
1666 rtx_code_label *do_main_label, *after_negate_label;
1667 rtx rop0, rop1;
1668 rop0 = gen_reg_rtx (mode);
1669 rop1 = gen_reg_rtx (mode);
1670 emit_move_insn (rop0, op0);
1671 emit_move_insn (rop1, op1);
1672 op0 = rop0;
1673 op1 = rop1;
1674 do_main_label = gen_label_rtx ();
1675 after_negate_label = gen_label_rtx ();
1676 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1677 OPTAB_LIB_WIDEN);
1678 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1679 NULL, after_negate_label, profile_probability::very_likely ());
1680 /* Both arguments negative here, negate them and continue with
1681 normal unsigned overflow checking multiplication. */
1682 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1683 NULL_RTX, false));
1684 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1685 NULL_RTX, false));
1686 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1687 the arguments. */
1688 arg0 = error_mark_node;
1689 arg1 = error_mark_node;
1690 emit_jump (do_main_label);
1691 emit_label (after_negate_label);
1692 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1693 OPTAB_LIB_WIDEN);
1694 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1695 NULL, do_main_label,
1696 profile_probability::very_likely ());
1697 /* One argument is negative here, the other positive. This
1698 overflows always, unless one of the arguments is 0. But
1699 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1700 is, thus we can keep do_main code oring in overflow as is. */
1701 if (pos_neg0 != 2)
1702 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1703 NULL, do_main_label,
1704 profile_probability::very_unlikely ());
1705 if (pos_neg1 != 2)
1706 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1707 NULL, do_main_label,
1708 profile_probability::very_unlikely ());
1709 expand_arith_set_overflow (lhs, target);
1710 emit_label (do_main_label);
1711 goto do_main;
1712 default:
1713 gcc_unreachable ();
1714 }
1715 }
1716
1717 do_main:
1718 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1719 sign = uns ? UNSIGNED : SIGNED;
1720 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1721 if (uns
1722 && (integer_pow2p (arg0) || integer_pow2p (arg1))
1723 && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1724 {
1725 /* Optimize unsigned multiplication by power of 2 constant
1726 using 2 shifts, one for result, one to extract the shifted
1727 out bits to see if they are all zero.
1728 Don't do this if optimizing for size and we have umulv4_optab,
1729 in that case assume multiplication will be shorter.
1730 This is heuristics based on the single target that provides
1731 umulv4 right now (i?86/x86_64), if further targets add it, this
1732 might need to be revisited.
1733 Cases where both operands are constant should be folded already
1734 during GIMPLE, and cases where one operand is constant but not
1735 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1736 below can be done without multiplication, just by shifts and adds,
1737 or we'd need to divide the result (and hope it actually doesn't
1738 really divide nor multiply) and compare the result of the division
1739 with the original operand. */
1740 rtx opn0 = op0;
1741 rtx opn1 = op1;
1742 tree argn0 = arg0;
1743 tree argn1 = arg1;
1744 if (integer_pow2p (arg0))
1745 {
1746 std::swap (opn0, opn1);
1747 std::swap (argn0, argn1);
1748 }
1749 int cnt = tree_log2 (argn1);
1750 if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1751 {
1752 rtx upper = const0_rtx;
1753 res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1754 if (cnt != 0)
1755 upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1756 GET_MODE_PRECISION (mode) - cnt,
1757 NULL_RTX, uns);
1758 do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
1759 NULL_RTX, NULL, done_label,
1760 profile_probability::very_likely ());
1761 goto do_error_label;
1762 }
1763 }
1764 if (icode != CODE_FOR_nothing)
1765 {
1766 class expand_operand ops[4];
1767 rtx_insn *last = get_last_insn ();
1768
1769 res = gen_reg_rtx (mode);
1770 create_output_operand (&ops[0], res, mode);
1771 create_input_operand (&ops[1], op0, mode);
1772 create_input_operand (&ops[2], op1, mode);
1773 create_fixed_operand (&ops[3], do_error);
1774 if (maybe_expand_insn (icode, 4, ops))
1775 {
1776 last = get_last_insn ();
1777 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1778 && JUMP_P (last)
1779 && any_condjump_p (last)
1780 && !find_reg_note (last, REG_BR_PROB, 0))
1781 add_reg_br_prob_note (last,
1782 profile_probability::very_unlikely ());
1783 emit_jump (done_label);
1784 }
1785 else
1786 {
1787 delete_insns_since (last);
1788 icode = CODE_FOR_nothing;
1789 }
1790 }
1791
1792 if (icode == CODE_FOR_nothing)
1793 {
1794 struct separate_ops ops;
1795 int prec = GET_MODE_PRECISION (mode);
1796 scalar_int_mode hmode, wmode;
1797 ops.op0 = make_tree (type, op0);
1798 ops.op1 = make_tree (type, op1);
1799 ops.op2 = NULL_TREE;
1800 ops.location = loc;
1801
1802 /* Optimize unsigned overflow check where we don't use the
1803 multiplication result, just whether overflow happened.
1804 If we can do MULT_HIGHPART_EXPR, that followed by
1805 comparison of the result against zero is cheapest.
1806 We'll still compute res, but it should be DCEd later. */
1807 use_operand_p use;
1808 gimple *use_stmt;
1809 if (!is_ubsan
1810 && lhs
1811 && uns
1812 && !(uns0_p && uns1_p && !unsr_p)
1813 && can_mult_highpart_p (mode, uns) == 1
1814 && single_imm_use (lhs, &use, &use_stmt)
1815 && is_gimple_assign (use_stmt)
1816 && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
1817 goto highpart;
1818
1819 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
1820 && targetm.scalar_mode_supported_p (wmode)
1821 && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
1822 {
1823 twoxwider:
1824 ops.code = WIDEN_MULT_EXPR;
1825 ops.type
1826 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
1827
1828 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1829 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1830 NULL_RTX, uns);
1831 hipart = convert_modes (mode, wmode, hipart, uns);
1832 res = convert_modes (mode, wmode, res, uns);
1833 if (uns)
1834 /* For the unsigned multiplication, there was overflow if
1835 HIPART is non-zero. */
1836 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1837 NULL_RTX, NULL, done_label,
1838 profile_probability::very_likely ());
1839 else
1840 {
1841 /* RES is used more than once, place it in a pseudo. */
1842 res = force_reg (mode, res);
1843
1844 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1845 NULL_RTX, 0);
1846 /* RES is low half of the double width result, HIPART
1847 the high half. There was overflow if
1848 HIPART is different from RES < 0 ? -1 : 0. */
1849 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1850 NULL_RTX, NULL, done_label,
1851 profile_probability::very_likely ());
1852 }
1853 }
1854 else if (can_mult_highpart_p (mode, uns) == 1)
1855 {
1856 highpart:
1857 ops.code = MULT_HIGHPART_EXPR;
1858 ops.type = type;
1859
1860 rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
1861 EXPAND_NORMAL);
1862 ops.code = MULT_EXPR;
1863 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1864 if (uns)
1865 /* For the unsigned multiplication, there was overflow if
1866 HIPART is non-zero. */
1867 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1868 NULL_RTX, NULL, done_label,
1869 profile_probability::very_likely ());
1870 else
1871 {
1872 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1873 NULL_RTX, 0);
1874 /* RES is low half of the double width result, HIPART
1875 the high half. There was overflow if
1876 HIPART is different from RES < 0 ? -1 : 0. */
1877 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1878 NULL_RTX, NULL, done_label,
1879 profile_probability::very_likely ());
1880 }
1881
1882 }
1883 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
1884 && 2 * GET_MODE_PRECISION (hmode) == prec)
1885 {
1886 rtx_code_label *large_op0 = gen_label_rtx ();
1887 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1888 rtx_code_label *one_small_one_large = gen_label_rtx ();
1889 rtx_code_label *both_ops_large = gen_label_rtx ();
1890 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1891 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
1892 rtx_code_label *do_overflow = gen_label_rtx ();
1893 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
1894
1895 unsigned int hprec = GET_MODE_PRECISION (hmode);
1896 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1897 NULL_RTX, uns);
1898 hipart0 = convert_modes (hmode, mode, hipart0, uns);
1899 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
1900 rtx signbit0 = const0_rtx;
1901 if (!uns)
1902 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1903 NULL_RTX, 0);
1904 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1905 NULL_RTX, uns);
1906 hipart1 = convert_modes (hmode, mode, hipart1, uns);
1907 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
1908 rtx signbit1 = const0_rtx;
1909 if (!uns)
1910 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1911 NULL_RTX, 0);
1912
1913 res = gen_reg_rtx (mode);
1914
1915 /* True if op0 resp. op1 are known to be in the range of
1916 halfstype. */
1917 bool op0_small_p = false;
1918 bool op1_small_p = false;
1919 /* True if op0 resp. op1 are known to have all zeros or all ones
1920 in the upper half of bits, but are not known to be
1921 op{0,1}_small_p. */
1922 bool op0_medium_p = false;
1923 bool op1_medium_p = false;
1924 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1925 nonnegative, 1 if unknown. */
1926 int op0_sign = 1;
1927 int op1_sign = 1;
1928
1929 if (pos_neg0 == 1)
1930 op0_sign = 0;
1931 else if (pos_neg0 == 2)
1932 op0_sign = -1;
1933 if (pos_neg1 == 1)
1934 op1_sign = 0;
1935 else if (pos_neg1 == 2)
1936 op1_sign = -1;
1937
1938 unsigned int mprec0 = prec;
1939 if (arg0 != error_mark_node)
1940 mprec0 = get_min_precision (arg0, sign);
1941 if (mprec0 <= hprec)
1942 op0_small_p = true;
1943 else if (!uns && mprec0 <= hprec + 1)
1944 op0_medium_p = true;
1945 unsigned int mprec1 = prec;
1946 if (arg1 != error_mark_node)
1947 mprec1 = get_min_precision (arg1, sign);
1948 if (mprec1 <= hprec)
1949 op1_small_p = true;
1950 else if (!uns && mprec1 <= hprec + 1)
1951 op1_medium_p = true;
1952
1953 int smaller_sign = 1;
1954 int larger_sign = 1;
1955 if (op0_small_p)
1956 {
1957 smaller_sign = op0_sign;
1958 larger_sign = op1_sign;
1959 }
1960 else if (op1_small_p)
1961 {
1962 smaller_sign = op1_sign;
1963 larger_sign = op0_sign;
1964 }
1965 else if (op0_sign == op1_sign)
1966 {
1967 smaller_sign = op0_sign;
1968 larger_sign = op0_sign;
1969 }
1970
1971 if (!op0_small_p)
1972 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
1973 NULL_RTX, NULL, large_op0,
1974 profile_probability::unlikely ());
1975
1976 if (!op1_small_p)
1977 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1978 NULL_RTX, NULL, small_op0_large_op1,
1979 profile_probability::unlikely ());
1980
1981 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1982 hmode to mode, the multiplication will never overflow. We can
1983 do just one hmode x hmode => mode widening multiplication. */
1984 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1985 ops.op0 = make_tree (halfstype, lopart0);
1986 ops.op1 = make_tree (halfstype, lopart1);
1987 ops.code = WIDEN_MULT_EXPR;
1988 ops.type = type;
1989 rtx thisres
1990 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1991 emit_move_insn (res, thisres);
1992 emit_jump (done_label);
1993
1994 emit_label (small_op0_large_op1);
1995
1996 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1997 but op1 is not, just swap the arguments and handle it as op1
1998 sign/zero extended, op0 not. */
1999 rtx larger = gen_reg_rtx (mode);
2000 rtx hipart = gen_reg_rtx (hmode);
2001 rtx lopart = gen_reg_rtx (hmode);
2002 emit_move_insn (larger, op1);
2003 emit_move_insn (hipart, hipart1);
2004 emit_move_insn (lopart, lopart0);
2005 emit_jump (one_small_one_large);
2006
2007 emit_label (large_op0);
2008
2009 if (!op1_small_p)
2010 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2011 NULL_RTX, NULL, both_ops_large,
2012 profile_probability::unlikely ());
2013
2014 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2015 but op0 is not, prepare larger, hipart and lopart pseudos and
2016 handle it together with small_op0_large_op1. */
2017 emit_move_insn (larger, op0);
2018 emit_move_insn (hipart, hipart0);
2019 emit_move_insn (lopart, lopart1);
2020
2021 emit_label (one_small_one_large);
2022
2023 /* lopart is the low part of the operand that is sign extended
2024 to mode, larger is the other operand, hipart is the
2025 high part of larger and lopart0 and lopart1 are the low parts
2026 of both operands.
2027 We perform lopart0 * lopart1 and lopart * hipart widening
2028 multiplications. */
2029 tree halfutype = build_nonstandard_integer_type (hprec, 1);
2030 ops.op0 = make_tree (halfutype, lopart0);
2031 ops.op1 = make_tree (halfutype, lopart1);
2032 rtx lo0xlo1
2033 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2034
2035 ops.op0 = make_tree (halfutype, lopart);
2036 ops.op1 = make_tree (halfutype, hipart);
2037 rtx loxhi = gen_reg_rtx (mode);
2038 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2039 emit_move_insn (loxhi, tem);
2040
2041 if (!uns)
2042 {
2043 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2044 if (larger_sign == 0)
2045 emit_jump (after_hipart_neg);
2046 else if (larger_sign != -1)
2047 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2048 NULL_RTX, NULL, after_hipart_neg,
2049 profile_probability::even ());
2050
2051 tem = convert_modes (mode, hmode, lopart, 1);
2052 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2053 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2054 1, OPTAB_WIDEN);
2055 emit_move_insn (loxhi, tem);
2056
2057 emit_label (after_hipart_neg);
2058
2059 /* if (lopart < 0) loxhi -= larger; */
2060 if (smaller_sign == 0)
2061 emit_jump (after_lopart_neg);
2062 else if (smaller_sign != -1)
2063 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2064 NULL_RTX, NULL, after_lopart_neg,
2065 profile_probability::even ());
2066
2067 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2068 1, OPTAB_WIDEN);
2069 emit_move_insn (loxhi, tem);
2070
2071 emit_label (after_lopart_neg);
2072 }
2073
2074 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2075 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2076 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2077 1, OPTAB_WIDEN);
2078 emit_move_insn (loxhi, tem);
2079
2080 /* if (loxhi >> (bitsize / 2)
2081 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2082 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2083 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2084 NULL_RTX, 0);
2085 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
2086 rtx signbitloxhi = const0_rtx;
2087 if (!uns)
2088 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2089 convert_modes (hmode, mode,
2090 loxhi, 0),
2091 hprec - 1, NULL_RTX, 0);
2092
2093 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2094 NULL_RTX, NULL, do_overflow,
2095 profile_probability::very_unlikely ());
2096
2097 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2098 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2099 NULL_RTX, 1);
2100 tem = convert_modes (mode, hmode,
2101 convert_modes (hmode, mode, lo0xlo1, 1), 1);
2102
2103 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2104 1, OPTAB_WIDEN);
2105 if (tem != res)
2106 emit_move_insn (res, tem);
2107 emit_jump (done_label);
2108
2109 emit_label (both_ops_large);
2110
2111 /* If both operands are large (not sign (!uns) or zero (uns)
2112 extended from hmode), then perform the full multiplication
2113 which will be the result of the operation.
2114 The only cases which don't overflow are for signed multiplication
2115 some cases where both hipart0 and highpart1 are 0 or -1.
2116 For unsigned multiplication when high parts are both non-zero
2117 this overflows always. */
2118 ops.code = MULT_EXPR;
2119 ops.op0 = make_tree (type, op0);
2120 ops.op1 = make_tree (type, op1);
2121 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2122 emit_move_insn (res, tem);
2123
2124 if (!uns)
2125 {
2126 if (!op0_medium_p)
2127 {
2128 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2129 NULL_RTX, 1, OPTAB_WIDEN);
2130 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2131 NULL_RTX, NULL, do_error,
2132 profile_probability::very_unlikely ());
2133 }
2134
2135 if (!op1_medium_p)
2136 {
2137 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2138 NULL_RTX, 1, OPTAB_WIDEN);
2139 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2140 NULL_RTX, NULL, do_error,
2141 profile_probability::very_unlikely ());
2142 }
2143
2144 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2145 the same, overflow happened if res is non-positive, if they
2146 are different, overflow happened if res is positive. */
2147 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2148 emit_jump (hipart_different);
2149 else if (op0_sign == 1 || op1_sign == 1)
2150 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2151 NULL_RTX, NULL, hipart_different,
2152 profile_probability::even ());
2153
2154 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2155 NULL_RTX, NULL, do_error,
2156 profile_probability::very_unlikely ());
2157 emit_jump (done_label);
2158
2159 emit_label (hipart_different);
2160
2161 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2162 NULL_RTX, NULL, do_error,
2163 profile_probability::very_unlikely ());
2164 emit_jump (done_label);
2165 }
2166
2167 emit_label (do_overflow);
2168
2169 /* Overflow, do full multiplication and fallthru into do_error. */
2170 ops.op0 = make_tree (type, op0);
2171 ops.op1 = make_tree (type, op1);
2172 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2173 emit_move_insn (res, tem);
2174 }
2175 else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
2176 && targetm.scalar_mode_supported_p (wmode))
2177 /* Even emitting a libcall is better than not detecting overflow
2178 at all. */
2179 goto twoxwider;
2180 else
2181 {
2182 gcc_assert (!is_ubsan);
2183 ops.code = MULT_EXPR;
2184 ops.type = type;
2185 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2186 emit_jump (done_label);
2187 }
2188 }
2189
2190 do_error_label:
2191 emit_label (do_error);
2192 if (is_ubsan)
2193 {
2194 /* Expand the ubsan builtin call. */
2195 push_temp_slots ();
2196 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2197 arg0, arg1, datap);
2198 expand_normal (fn);
2199 pop_temp_slots ();
2200 do_pending_stack_adjust ();
2201 }
2202 else if (lhs)
2203 expand_arith_set_overflow (lhs, target);
2204
2205 /* We're done. */
2206 emit_label (done_label);
2207
2208 /* u1 * u2 -> sr */
2209 if (uns0_p && uns1_p && !unsr_p)
2210 {
2211 rtx_code_label *all_done_label = gen_label_rtx ();
2212 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2213 NULL, all_done_label, profile_probability::very_likely ());
2214 expand_arith_set_overflow (lhs, target);
2215 emit_label (all_done_label);
2216 }
2217
2218 /* s1 * u2 -> sr */
2219 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2220 {
2221 rtx_code_label *all_done_label = gen_label_rtx ();
2222 rtx_code_label *set_noovf = gen_label_rtx ();
2223 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2224 NULL, all_done_label, profile_probability::very_likely ());
2225 expand_arith_set_overflow (lhs, target);
2226 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2227 NULL, set_noovf, profile_probability::very_likely ());
2228 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2229 NULL, all_done_label, profile_probability::very_unlikely ());
2230 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2231 all_done_label, profile_probability::very_unlikely ());
2232 emit_label (set_noovf);
2233 write_complex_part (target, const0_rtx, true);
2234 emit_label (all_done_label);
2235 }
2236
2237 if (lhs)
2238 {
2239 if (is_ubsan)
2240 expand_ubsan_result_store (target, res);
2241 else
2242 expand_arith_overflow_result_store (lhs, target, mode, res);
2243 }
2244 flag_trapv = save_flag_trapv;
2245 }
2246
2247 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2248
2249 static void
expand_vector_ubsan_overflow(location_t loc,enum tree_code code,tree lhs,tree arg0,tree arg1)2250 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2251 tree arg0, tree arg1)
2252 {
2253 poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2254 rtx_code_label *loop_lab = NULL;
2255 rtx cntvar = NULL_RTX;
2256 tree cntv = NULL_TREE;
2257 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2258 tree sz = TYPE_SIZE (eltype);
2259 tree data = NULL_TREE;
2260 tree resv = NULL_TREE;
2261 rtx lhsr = NULL_RTX;
2262 rtx resvr = NULL_RTX;
2263 unsigned HOST_WIDE_INT const_cnt = 0;
2264 bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
2265 int save_flag_trapv = flag_trapv;
2266
2267 /* We don't want any __mulv?i3 etc. calls from the expansion of
2268 these internal functions, so disable -ftrapv temporarily. */
2269 flag_trapv = 0;
2270 if (lhs)
2271 {
2272 optab op;
2273 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2274 if (!VECTOR_MODE_P (GET_MODE (lhsr))
2275 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2276 optab_default)) == unknown_optab
2277 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2278 == CODE_FOR_nothing))
2279 {
2280 if (MEM_P (lhsr))
2281 resv = make_tree (TREE_TYPE (lhs), lhsr);
2282 else
2283 {
2284 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2285 resv = make_tree (TREE_TYPE (lhs), resvr);
2286 }
2287 }
2288 }
2289 if (use_loop_p)
2290 {
2291 do_pending_stack_adjust ();
2292 loop_lab = gen_label_rtx ();
2293 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2294 cntv = make_tree (sizetype, cntvar);
2295 emit_move_insn (cntvar, const0_rtx);
2296 emit_label (loop_lab);
2297 }
2298 if (TREE_CODE (arg0) != VECTOR_CST)
2299 {
2300 rtx arg0r = expand_normal (arg0);
2301 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2302 }
2303 if (TREE_CODE (arg1) != VECTOR_CST)
2304 {
2305 rtx arg1r = expand_normal (arg1);
2306 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2307 }
2308 for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2309 {
2310 tree op0, op1, res = NULL_TREE;
2311 if (use_loop_p)
2312 {
2313 tree atype = build_array_type_nelts (eltype, cnt);
2314 op0 = uniform_vector_p (arg0);
2315 if (op0 == NULL_TREE)
2316 {
2317 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2318 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
2319 NULL_TREE, NULL_TREE);
2320 }
2321 op1 = uniform_vector_p (arg1);
2322 if (op1 == NULL_TREE)
2323 {
2324 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2325 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
2326 NULL_TREE, NULL_TREE);
2327 }
2328 if (resv)
2329 {
2330 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2331 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
2332 NULL_TREE, NULL_TREE);
2333 }
2334 }
2335 else
2336 {
2337 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2338 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2339 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2340 if (resv)
2341 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2342 bitpos);
2343 }
2344 switch (code)
2345 {
2346 case PLUS_EXPR:
2347 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
2348 false, false, false, true, &data);
2349 break;
2350 case MINUS_EXPR:
2351 if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2352 expand_neg_overflow (loc, res, op1, true, &data);
2353 else
2354 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
2355 false, false, false, true, &data);
2356 break;
2357 case MULT_EXPR:
2358 expand_mul_overflow (loc, res, op0, op1, false, false, false,
2359 true, &data);
2360 break;
2361 default:
2362 gcc_unreachable ();
2363 }
2364 }
2365 if (use_loop_p)
2366 {
2367 struct separate_ops ops;
2368 ops.code = PLUS_EXPR;
2369 ops.type = TREE_TYPE (cntv);
2370 ops.op0 = cntv;
2371 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2372 ops.op2 = NULL_TREE;
2373 ops.location = loc;
2374 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2375 EXPAND_NORMAL);
2376 if (ret != cntvar)
2377 emit_move_insn (cntvar, ret);
2378 rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2379 do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2380 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2381 profile_probability::very_likely ());
2382 }
2383 if (lhs && resv == NULL_TREE)
2384 {
2385 struct separate_ops ops;
2386 ops.code = code;
2387 ops.type = TREE_TYPE (arg0);
2388 ops.op0 = arg0;
2389 ops.op1 = arg1;
2390 ops.op2 = NULL_TREE;
2391 ops.location = loc;
2392 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2393 EXPAND_NORMAL);
2394 if (ret != lhsr)
2395 emit_move_insn (lhsr, ret);
2396 }
2397 else if (resvr)
2398 emit_move_insn (lhsr, resvr);
2399 flag_trapv = save_flag_trapv;
2400 }
2401
2402 /* Expand UBSAN_CHECK_ADD call STMT. */
2403
2404 static void
expand_UBSAN_CHECK_ADD(internal_fn,gcall * stmt)2405 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2406 {
2407 location_t loc = gimple_location (stmt);
2408 tree lhs = gimple_call_lhs (stmt);
2409 tree arg0 = gimple_call_arg (stmt, 0);
2410 tree arg1 = gimple_call_arg (stmt, 1);
2411 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2412 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
2413 else
2414 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
2415 false, false, false, true, NULL);
2416 }
2417
2418 /* Expand UBSAN_CHECK_SUB call STMT. */
2419
2420 static void
expand_UBSAN_CHECK_SUB(internal_fn,gcall * stmt)2421 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2422 {
2423 location_t loc = gimple_location (stmt);
2424 tree lhs = gimple_call_lhs (stmt);
2425 tree arg0 = gimple_call_arg (stmt, 0);
2426 tree arg1 = gimple_call_arg (stmt, 1);
2427 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2428 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
2429 else if (integer_zerop (arg0))
2430 expand_neg_overflow (loc, lhs, arg1, true, NULL);
2431 else
2432 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
2433 false, false, false, true, NULL);
2434 }
2435
2436 /* Expand UBSAN_CHECK_MUL call STMT. */
2437
2438 static void
expand_UBSAN_CHECK_MUL(internal_fn,gcall * stmt)2439 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2440 {
2441 location_t loc = gimple_location (stmt);
2442 tree lhs = gimple_call_lhs (stmt);
2443 tree arg0 = gimple_call_arg (stmt, 0);
2444 tree arg1 = gimple_call_arg (stmt, 1);
2445 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2446 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
2447 else
2448 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
2449 NULL);
2450 }
2451
2452 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2453
2454 static void
expand_arith_overflow(enum tree_code code,gimple * stmt)2455 expand_arith_overflow (enum tree_code code, gimple *stmt)
2456 {
2457 tree lhs = gimple_call_lhs (stmt);
2458 if (lhs == NULL_TREE)
2459 return;
2460 tree arg0 = gimple_call_arg (stmt, 0);
2461 tree arg1 = gimple_call_arg (stmt, 1);
2462 tree type = TREE_TYPE (TREE_TYPE (lhs));
2463 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2464 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2465 int unsr_p = TYPE_UNSIGNED (type);
2466 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2467 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2468 int precres = TYPE_PRECISION (type);
2469 location_t loc = gimple_location (stmt);
2470 if (!uns0_p && get_range_pos_neg (arg0) == 1)
2471 uns0_p = true;
2472 if (!uns1_p && get_range_pos_neg (arg1) == 1)
2473 uns1_p = true;
2474 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
2475 prec0 = MIN (prec0, pr);
2476 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
2477 prec1 = MIN (prec1, pr);
2478 int save_flag_trapv = flag_trapv;
2479
2480 /* We don't want any __mulv?i3 etc. calls from the expansion of
2481 these internal functions, so disable -ftrapv temporarily. */
2482 flag_trapv = 0;
2483 /* If uns0_p && uns1_p, precop is minimum needed precision
2484 of unsigned type to hold the exact result, otherwise
2485 precop is minimum needed precision of signed type to
2486 hold the exact result. */
2487 int precop;
2488 if (code == MULT_EXPR)
2489 precop = prec0 + prec1 + (uns0_p != uns1_p);
2490 else
2491 {
2492 if (uns0_p == uns1_p)
2493 precop = MAX (prec0, prec1) + 1;
2494 else if (uns0_p)
2495 precop = MAX (prec0 + 1, prec1) + 1;
2496 else
2497 precop = MAX (prec0, prec1 + 1) + 1;
2498 }
2499 int orig_precres = precres;
2500
2501 do
2502 {
2503 if ((uns0_p && uns1_p)
2504 ? ((precop + !unsr_p) <= precres
2505 /* u1 - u2 -> ur can overflow, no matter what precision
2506 the result has. */
2507 && (code != MINUS_EXPR || !unsr_p))
2508 : (!unsr_p && precop <= precres))
2509 {
2510 /* The infinity precision result will always fit into result. */
2511 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2512 write_complex_part (target, const0_rtx, true);
2513 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2514 struct separate_ops ops;
2515 ops.code = code;
2516 ops.type = type;
2517 ops.op0 = fold_convert_loc (loc, type, arg0);
2518 ops.op1 = fold_convert_loc (loc, type, arg1);
2519 ops.op2 = NULL_TREE;
2520 ops.location = loc;
2521 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2522 expand_arith_overflow_result_store (lhs, target, mode, tem);
2523 flag_trapv = save_flag_trapv;
2524 return;
2525 }
2526
2527 /* For operations with low precision, if target doesn't have them, start
2528 with precres widening right away, otherwise do it only if the most
2529 simple cases can't be used. */
2530 const int min_precision = targetm.min_arithmetic_precision ();
2531 if (orig_precres == precres && precres < min_precision)
2532 ;
2533 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2534 && prec1 <= precres)
2535 || ((!uns0_p || !uns1_p) && !unsr_p
2536 && prec0 + uns0_p <= precres
2537 && prec1 + uns1_p <= precres))
2538 {
2539 arg0 = fold_convert_loc (loc, type, arg0);
2540 arg1 = fold_convert_loc (loc, type, arg1);
2541 switch (code)
2542 {
2543 case MINUS_EXPR:
2544 if (integer_zerop (arg0) && !unsr_p)
2545 {
2546 expand_neg_overflow (loc, lhs, arg1, false, NULL);
2547 flag_trapv = save_flag_trapv;
2548 return;
2549 }
2550 /* FALLTHRU */
2551 case PLUS_EXPR:
2552 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2553 unsr_p, unsr_p, false, NULL);
2554 flag_trapv = save_flag_trapv;
2555 return;
2556 case MULT_EXPR:
2557 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2558 unsr_p, unsr_p, false, NULL);
2559 flag_trapv = save_flag_trapv;
2560 return;
2561 default:
2562 gcc_unreachable ();
2563 }
2564 }
2565
2566 /* For sub-word operations, retry with a wider type first. */
2567 if (orig_precres == precres && precop <= BITS_PER_WORD)
2568 {
2569 int p = MAX (min_precision, precop);
2570 scalar_int_mode m = smallest_int_mode_for_size (p);
2571 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2572 uns0_p && uns1_p
2573 && unsr_p);
2574 p = TYPE_PRECISION (optype);
2575 if (p > precres)
2576 {
2577 precres = p;
2578 unsr_p = TYPE_UNSIGNED (optype);
2579 type = optype;
2580 continue;
2581 }
2582 }
2583
2584 if (prec0 <= precres && prec1 <= precres)
2585 {
2586 tree types[2];
2587 if (unsr_p)
2588 {
2589 types[0] = build_nonstandard_integer_type (precres, 0);
2590 types[1] = type;
2591 }
2592 else
2593 {
2594 types[0] = type;
2595 types[1] = build_nonstandard_integer_type (precres, 1);
2596 }
2597 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2598 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2599 if (code != MULT_EXPR)
2600 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2601 uns0_p, uns1_p, false, NULL);
2602 else
2603 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2604 uns0_p, uns1_p, false, NULL);
2605 flag_trapv = save_flag_trapv;
2606 return;
2607 }
2608
2609 /* Retry with a wider type. */
2610 if (orig_precres == precres)
2611 {
2612 int p = MAX (prec0, prec1);
2613 scalar_int_mode m = smallest_int_mode_for_size (p);
2614 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2615 uns0_p && uns1_p
2616 && unsr_p);
2617 p = TYPE_PRECISION (optype);
2618 if (p > precres)
2619 {
2620 precres = p;
2621 unsr_p = TYPE_UNSIGNED (optype);
2622 type = optype;
2623 continue;
2624 }
2625 }
2626
2627 gcc_unreachable ();
2628 }
2629 while (1);
2630 }
2631
2632 /* Expand ADD_OVERFLOW STMT. */
2633
2634 static void
expand_ADD_OVERFLOW(internal_fn,gcall * stmt)2635 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2636 {
2637 expand_arith_overflow (PLUS_EXPR, stmt);
2638 }
2639
2640 /* Expand SUB_OVERFLOW STMT. */
2641
2642 static void
expand_SUB_OVERFLOW(internal_fn,gcall * stmt)2643 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2644 {
2645 expand_arith_overflow (MINUS_EXPR, stmt);
2646 }
2647
2648 /* Expand MUL_OVERFLOW STMT. */
2649
2650 static void
expand_MUL_OVERFLOW(internal_fn,gcall * stmt)2651 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2652 {
2653 expand_arith_overflow (MULT_EXPR, stmt);
2654 }
2655
2656 /* This should get folded in tree-vectorizer.cc. */
2657
2658 static void
expand_LOOP_VECTORIZED(internal_fn,gcall *)2659 expand_LOOP_VECTORIZED (internal_fn, gcall *)
2660 {
2661 gcc_unreachable ();
2662 }
2663
2664 /* This should get folded in tree-vectorizer.cc. */
2665
2666 static void
expand_LOOP_DIST_ALIAS(internal_fn,gcall *)2667 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2668 {
2669 gcc_unreachable ();
2670 }
2671
2672 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2673 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2674
2675 static tree
expand_call_mem_ref(tree type,gcall * stmt,int index)2676 expand_call_mem_ref (tree type, gcall *stmt, int index)
2677 {
2678 tree addr = gimple_call_arg (stmt, index);
2679 tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2680 unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
2681 if (TYPE_ALIGN (type) != align)
2682 type = build_aligned_type (type, align);
2683
2684 tree tmp = addr;
2685 if (TREE_CODE (tmp) == SSA_NAME)
2686 {
2687 gimple *def = get_gimple_for_ssa_name (tmp);
2688 if (def && gimple_assign_single_p (def))
2689 tmp = gimple_assign_rhs1 (def);
2690 }
2691
2692 if (TREE_CODE (tmp) == ADDR_EXPR)
2693 {
2694 tree mem = TREE_OPERAND (tmp, 0);
2695 if (TREE_CODE (mem) == TARGET_MEM_REF
2696 && types_compatible_p (TREE_TYPE (mem), type))
2697 {
2698 tree offset = TMR_OFFSET (mem);
2699 if (type != TREE_TYPE (mem)
2700 || alias_ptr_type != TREE_TYPE (offset)
2701 || !integer_zerop (offset))
2702 {
2703 mem = copy_node (mem);
2704 TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
2705 wi::to_poly_wide (offset));
2706 TREE_TYPE (mem) = type;
2707 }
2708 return mem;
2709 }
2710 }
2711
2712 return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2713 }
2714
2715 /* Expand MASK_LOAD{,_LANES} or LEN_LOAD call STMT using optab OPTAB. */
2716
2717 static void
expand_partial_load_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2718 expand_partial_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2719 {
2720 class expand_operand ops[4];
2721 tree type, lhs, rhs, maskt, biast;
2722 rtx mem, target, mask, bias;
2723 insn_code icode;
2724
2725 maskt = gimple_call_arg (stmt, 2);
2726 lhs = gimple_call_lhs (stmt);
2727 if (lhs == NULL_TREE)
2728 return;
2729 type = TREE_TYPE (lhs);
2730 rhs = expand_call_mem_ref (type, stmt, 0);
2731
2732 if (optab == vec_mask_load_lanes_optab)
2733 icode = get_multi_vector_move (type, optab);
2734 else if (optab == len_load_optab)
2735 icode = direct_optab_handler (optab, TYPE_MODE (type));
2736 else
2737 icode = convert_optab_handler (optab, TYPE_MODE (type),
2738 TYPE_MODE (TREE_TYPE (maskt)));
2739
2740 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2741 gcc_assert (MEM_P (mem));
2742 mask = expand_normal (maskt);
2743 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2744 create_output_operand (&ops[0], target, TYPE_MODE (type));
2745 create_fixed_operand (&ops[1], mem);
2746 if (optab == len_load_optab)
2747 {
2748 create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
2749 TYPE_UNSIGNED (TREE_TYPE (maskt)));
2750 biast = gimple_call_arg (stmt, 3);
2751 bias = expand_normal (biast);
2752 create_input_operand (&ops[3], bias, QImode);
2753 expand_insn (icode, 4, ops);
2754 }
2755 else
2756 {
2757 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2758 expand_insn (icode, 3, ops);
2759 }
2760
2761 if (!rtx_equal_p (target, ops[0].value))
2762 emit_move_insn (target, ops[0].value);
2763 }
2764
2765 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2766 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2767 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2768
2769 /* Expand MASK_STORE{,_LANES} or LEN_STORE call STMT using optab OPTAB. */
2770
2771 static void
expand_partial_store_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2772 expand_partial_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2773 {
2774 class expand_operand ops[4];
2775 tree type, lhs, rhs, maskt, biast;
2776 rtx mem, reg, mask, bias;
2777 insn_code icode;
2778
2779 maskt = gimple_call_arg (stmt, 2);
2780 rhs = gimple_call_arg (stmt, 3);
2781 type = TREE_TYPE (rhs);
2782 lhs = expand_call_mem_ref (type, stmt, 0);
2783
2784 if (optab == vec_mask_store_lanes_optab)
2785 icode = get_multi_vector_move (type, optab);
2786 else if (optab == len_store_optab)
2787 icode = direct_optab_handler (optab, TYPE_MODE (type));
2788 else
2789 icode = convert_optab_handler (optab, TYPE_MODE (type),
2790 TYPE_MODE (TREE_TYPE (maskt)));
2791
2792 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2793 gcc_assert (MEM_P (mem));
2794 mask = expand_normal (maskt);
2795 reg = expand_normal (rhs);
2796 create_fixed_operand (&ops[0], mem);
2797 create_input_operand (&ops[1], reg, TYPE_MODE (type));
2798 if (optab == len_store_optab)
2799 {
2800 create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
2801 TYPE_UNSIGNED (TREE_TYPE (maskt)));
2802 biast = gimple_call_arg (stmt, 4);
2803 bias = expand_normal (biast);
2804 create_input_operand (&ops[3], bias, QImode);
2805 expand_insn (icode, 4, ops);
2806 }
2807 else
2808 {
2809 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2810 expand_insn (icode, 3, ops);
2811 }
2812 }
2813
2814 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
2815 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2816 #define expand_len_store_optab_fn expand_partial_store_optab_fn
2817
2818 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2819 The expansion of STMT happens based on OPTAB table associated. */
2820
2821 static void
expand_vec_cond_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2822 expand_vec_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2823 {
2824 class expand_operand ops[6];
2825 insn_code icode;
2826 tree lhs = gimple_call_lhs (stmt);
2827 tree op0a = gimple_call_arg (stmt, 0);
2828 tree op0b = gimple_call_arg (stmt, 1);
2829 tree op1 = gimple_call_arg (stmt, 2);
2830 tree op2 = gimple_call_arg (stmt, 3);
2831 enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (stmt, 4));
2832
2833 tree vec_cond_type = TREE_TYPE (lhs);
2834 tree op_mode = TREE_TYPE (op0a);
2835 bool unsignedp = TYPE_UNSIGNED (op_mode);
2836
2837 machine_mode mode = TYPE_MODE (vec_cond_type);
2838 machine_mode cmp_op_mode = TYPE_MODE (op_mode);
2839
2840 icode = convert_optab_handler (optab, mode, cmp_op_mode);
2841 rtx comparison
2842 = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, icode, 4);
2843 rtx rtx_op1 = expand_normal (op1);
2844 rtx rtx_op2 = expand_normal (op2);
2845
2846 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2847 create_output_operand (&ops[0], target, mode);
2848 create_input_operand (&ops[1], rtx_op1, mode);
2849 create_input_operand (&ops[2], rtx_op2, mode);
2850 create_fixed_operand (&ops[3], comparison);
2851 create_fixed_operand (&ops[4], XEXP (comparison, 0));
2852 create_fixed_operand (&ops[5], XEXP (comparison, 1));
2853 expand_insn (icode, 6, ops);
2854 if (!rtx_equal_p (ops[0].value, target))
2855 emit_move_insn (target, ops[0].value);
2856 }
2857
2858 /* Expand VCOND_MASK optab internal function.
2859 The expansion of STMT happens based on OPTAB table associated. */
2860
2861 static void
expand_vec_cond_mask_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2862 expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2863 {
2864 class expand_operand ops[4];
2865
2866 tree lhs = gimple_call_lhs (stmt);
2867 tree op0 = gimple_call_arg (stmt, 0);
2868 tree op1 = gimple_call_arg (stmt, 1);
2869 tree op2 = gimple_call_arg (stmt, 2);
2870 tree vec_cond_type = TREE_TYPE (lhs);
2871
2872 machine_mode mode = TYPE_MODE (vec_cond_type);
2873 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
2874 enum insn_code icode = convert_optab_handler (optab, mode, mask_mode);
2875 rtx mask, rtx_op1, rtx_op2;
2876
2877 gcc_assert (icode != CODE_FOR_nothing);
2878
2879 mask = expand_normal (op0);
2880 rtx_op1 = expand_normal (op1);
2881 rtx_op2 = expand_normal (op2);
2882
2883 mask = force_reg (mask_mode, mask);
2884 rtx_op1 = force_reg (mode, rtx_op1);
2885
2886 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2887 create_output_operand (&ops[0], target, mode);
2888 create_input_operand (&ops[1], rtx_op1, mode);
2889 create_input_operand (&ops[2], rtx_op2, mode);
2890 create_input_operand (&ops[3], mask, mask_mode);
2891 expand_insn (icode, 4, ops);
2892 if (!rtx_equal_p (ops[0].value, target))
2893 emit_move_insn (target, ops[0].value);
2894 }
2895
2896 /* Expand VEC_SET internal functions. */
2897
2898 static void
expand_vec_set_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2899 expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2900 {
2901 tree lhs = gimple_call_lhs (stmt);
2902 tree op0 = gimple_call_arg (stmt, 0);
2903 tree op1 = gimple_call_arg (stmt, 1);
2904 tree op2 = gimple_call_arg (stmt, 2);
2905 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2906 rtx src = expand_normal (op0);
2907
2908 machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
2909 scalar_mode innermode = GET_MODE_INNER (outermode);
2910
2911 rtx value = expand_normal (op1);
2912 rtx pos = expand_normal (op2);
2913
2914 class expand_operand ops[3];
2915 enum insn_code icode = optab_handler (optab, outermode);
2916
2917 if (icode != CODE_FOR_nothing)
2918 {
2919 rtx temp = gen_reg_rtx (outermode);
2920 emit_move_insn (temp, src);
2921
2922 create_fixed_operand (&ops[0], temp);
2923 create_input_operand (&ops[1], value, innermode);
2924 create_convert_operand_from (&ops[2], pos, TYPE_MODE (TREE_TYPE (op2)),
2925 true);
2926 if (maybe_expand_insn (icode, 3, ops))
2927 {
2928 emit_move_insn (target, temp);
2929 return;
2930 }
2931 }
2932 gcc_unreachable ();
2933 }
2934
2935 static void
expand_ABNORMAL_DISPATCHER(internal_fn,gcall *)2936 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
2937 {
2938 }
2939
2940 static void
expand_BUILTIN_EXPECT(internal_fn,gcall * stmt)2941 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
2942 {
2943 /* When guessing was done, the hints should be already stripped away. */
2944 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
2945
2946 rtx target;
2947 tree lhs = gimple_call_lhs (stmt);
2948 if (lhs)
2949 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2950 else
2951 target = const0_rtx;
2952 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
2953 if (lhs && val != target)
2954 emit_move_insn (target, val);
2955 }
2956
2957 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2958 should never be called. */
2959
2960 static void
expand_VA_ARG(internal_fn,gcall *)2961 expand_VA_ARG (internal_fn, gcall *)
2962 {
2963 gcc_unreachable ();
2964 }
2965
2966 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2967 dummy function should never be called. */
2968
2969 static void
expand_VEC_CONVERT(internal_fn,gcall *)2970 expand_VEC_CONVERT (internal_fn, gcall *)
2971 {
2972 gcc_unreachable ();
2973 }
2974
2975 /* Expand IFN_RAWMEMCHAR internal function. */
2976
2977 void
expand_RAWMEMCHR(internal_fn,gcall * stmt)2978 expand_RAWMEMCHR (internal_fn, gcall *stmt)
2979 {
2980 expand_operand ops[3];
2981
2982 tree lhs = gimple_call_lhs (stmt);
2983 if (!lhs)
2984 return;
2985 machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
2986 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2987 create_output_operand (&ops[0], lhs_rtx, lhs_mode);
2988
2989 tree mem = gimple_call_arg (stmt, 0);
2990 rtx mem_rtx = get_memory_rtx (mem, NULL);
2991 create_fixed_operand (&ops[1], mem_rtx);
2992
2993 tree pattern = gimple_call_arg (stmt, 1);
2994 machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
2995 rtx pattern_rtx = expand_normal (pattern);
2996 create_input_operand (&ops[2], pattern_rtx, mode);
2997
2998 insn_code icode = direct_optab_handler (rawmemchr_optab, mode);
2999
3000 expand_insn (icode, 3, ops);
3001 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3002 emit_move_insn (lhs_rtx, ops[0].value);
3003 }
3004
3005 /* Expand the IFN_UNIQUE function according to its first argument. */
3006
3007 static void
expand_UNIQUE(internal_fn,gcall * stmt)3008 expand_UNIQUE (internal_fn, gcall *stmt)
3009 {
3010 rtx pattern = NULL_RTX;
3011 enum ifn_unique_kind kind
3012 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
3013
3014 switch (kind)
3015 {
3016 default:
3017 gcc_unreachable ();
3018
3019 case IFN_UNIQUE_UNSPEC:
3020 if (targetm.have_unique ())
3021 pattern = targetm.gen_unique ();
3022 break;
3023
3024 case IFN_UNIQUE_OACC_FORK:
3025 case IFN_UNIQUE_OACC_JOIN:
3026 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3027 {
3028 tree lhs = gimple_call_lhs (stmt);
3029 rtx target = const0_rtx;
3030
3031 if (lhs)
3032 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3033
3034 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
3035 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
3036
3037 if (kind == IFN_UNIQUE_OACC_FORK)
3038 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3039 else
3040 pattern = targetm.gen_oacc_join (target, data_dep, axis);
3041 }
3042 else
3043 gcc_unreachable ();
3044 break;
3045 }
3046
3047 if (pattern)
3048 emit_insn (pattern);
3049 }
3050
3051 /* Expand the IFN_DEFERRED_INIT function:
3052 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3053
3054 Initialize the LHS with zero/pattern according to its second argument
3055 INIT_TYPE:
3056 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3057 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3058 to initialize;
3059 The LHS variable is initialized including paddings.
3060 The reasons to choose 0xFE for pattern initialization are:
3061 1. It is a non-canonical virtual address on x86_64, and at the
3062 high end of the i386 kernel address space.
3063 2. It is a very large float value (-1.694739530317379e+38).
3064 3. It is also an unusual number for integers. */
3065 #define INIT_PATTERN_VALUE 0xFE
3066 static void
expand_DEFERRED_INIT(internal_fn,gcall * stmt)3067 expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3068 {
3069 tree lhs = gimple_call_lhs (stmt);
3070 tree var_size = gimple_call_arg (stmt, 0);
3071 enum auto_init_type init_type
3072 = (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3073 bool reg_lhs = true;
3074
3075 tree var_type = TREE_TYPE (lhs);
3076 gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3077
3078 if (TREE_CODE (lhs) == SSA_NAME)
3079 reg_lhs = true;
3080 else
3081 {
3082 tree lhs_base = lhs;
3083 while (handled_component_p (lhs_base))
3084 lhs_base = TREE_OPERAND (lhs_base, 0);
3085 reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3086 || non_mem_decl_p (lhs_base));
3087 /* If this expands to a register and the underlying decl is wrapped in
3088 a MEM_REF that just serves as an access type change expose the decl
3089 if it is of correct size. This avoids a situation as in PR103271
3090 if the target does not support a direct move to the registers mode. */
3091 if (reg_lhs
3092 && TREE_CODE (lhs_base) == MEM_REF
3093 && TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3094 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3095 && integer_zerop (TREE_OPERAND (lhs_base, 1))
3096 && tree_fits_uhwi_p (var_size)
3097 && tree_int_cst_equal
3098 (var_size,
3099 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3100 {
3101 lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3102 var_type = TREE_TYPE (lhs);
3103 }
3104 }
3105
3106 if (!reg_lhs)
3107 {
3108 /* If the variable is not in register, expand to a memset
3109 to initialize it. */
3110 mark_addressable (lhs);
3111 tree var_addr = build_fold_addr_expr (lhs);
3112
3113 tree value = (init_type == AUTO_INIT_PATTERN)
3114 ? build_int_cst (integer_type_node,
3115 INIT_PATTERN_VALUE)
3116 : integer_zero_node;
3117 tree m_call = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET),
3118 3, var_addr, value, var_size);
3119 /* Expand this memset call. */
3120 expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3121 }
3122 else
3123 {
3124 /* If this variable is in a register use expand_assignment.
3125 For boolean scalars force zero-init. */
3126 tree init;
3127 scalar_int_mode var_mode;
3128 if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3129 && tree_fits_uhwi_p (var_size)
3130 && (init_type == AUTO_INIT_PATTERN
3131 || !is_gimple_reg_type (var_type))
3132 && int_mode_for_size (tree_to_uhwi (var_size) * BITS_PER_UNIT,
3133 0).exists (&var_mode)
3134 && have_insn_for (SET, var_mode))
3135 {
3136 unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3137 unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3138 memset (buf, (init_type == AUTO_INIT_PATTERN
3139 ? INIT_PATTERN_VALUE : 0), total_bytes);
3140 tree itype = build_nonstandard_integer_type
3141 (total_bytes * BITS_PER_UNIT, 1);
3142 wide_int w = wi::from_buffer (buf, total_bytes);
3143 init = wide_int_to_tree (itype, w);
3144 /* Pun the LHS to make sure its type has constant size
3145 unless it is an SSA name where that's already known. */
3146 if (TREE_CODE (lhs) != SSA_NAME)
3147 lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3148 else
3149 init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3150 }
3151 else
3152 /* Use zero-init also for variable-length sizes. */
3153 init = build_zero_cst (var_type);
3154
3155 expand_assignment (lhs, init, false);
3156 }
3157 }
3158
3159 /* The size of an OpenACC compute dimension. */
3160
3161 static void
expand_GOACC_DIM_SIZE(internal_fn,gcall * stmt)3162 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3163 {
3164 tree lhs = gimple_call_lhs (stmt);
3165
3166 if (!lhs)
3167 return;
3168
3169 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3170 if (targetm.have_oacc_dim_size ())
3171 {
3172 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3173 VOIDmode, EXPAND_NORMAL);
3174 emit_insn (targetm.gen_oacc_dim_size (target, dim));
3175 }
3176 else
3177 emit_move_insn (target, GEN_INT (1));
3178 }
3179
3180 /* The position of an OpenACC execution engine along one compute axis. */
3181
3182 static void
expand_GOACC_DIM_POS(internal_fn,gcall * stmt)3183 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3184 {
3185 tree lhs = gimple_call_lhs (stmt);
3186
3187 if (!lhs)
3188 return;
3189
3190 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3191 if (targetm.have_oacc_dim_pos ())
3192 {
3193 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3194 VOIDmode, EXPAND_NORMAL);
3195 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3196 }
3197 else
3198 emit_move_insn (target, const0_rtx);
3199 }
3200
3201 /* This is expanded by oacc_device_lower pass. */
3202
3203 static void
expand_GOACC_LOOP(internal_fn,gcall *)3204 expand_GOACC_LOOP (internal_fn, gcall *)
3205 {
3206 gcc_unreachable ();
3207 }
3208
3209 /* This is expanded by oacc_device_lower pass. */
3210
3211 static void
expand_GOACC_REDUCTION(internal_fn,gcall *)3212 expand_GOACC_REDUCTION (internal_fn, gcall *)
3213 {
3214 gcc_unreachable ();
3215 }
3216
3217 /* This is expanded by oacc_device_lower pass. */
3218
3219 static void
expand_GOACC_TILE(internal_fn,gcall *)3220 expand_GOACC_TILE (internal_fn, gcall *)
3221 {
3222 gcc_unreachable ();
3223 }
3224
3225 /* Set errno to EDOM. */
3226
3227 static void
expand_SET_EDOM(internal_fn,gcall *)3228 expand_SET_EDOM (internal_fn, gcall *)
3229 {
3230 #ifdef TARGET_EDOM
3231 #ifdef GEN_ERRNO_RTX
3232 rtx errno_rtx = GEN_ERRNO_RTX;
3233 #else
3234 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3235 #endif
3236 emit_move_insn (errno_rtx,
3237 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3238 #else
3239 gcc_unreachable ();
3240 #endif
3241 }
3242
3243 /* Expand atomic bit test and set. */
3244
3245 static void
expand_ATOMIC_BIT_TEST_AND_SET(internal_fn,gcall * call)3246 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3247 {
3248 expand_ifn_atomic_bit_test_and (call);
3249 }
3250
3251 /* Expand atomic bit test and complement. */
3252
3253 static void
expand_ATOMIC_BIT_TEST_AND_COMPLEMENT(internal_fn,gcall * call)3254 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3255 {
3256 expand_ifn_atomic_bit_test_and (call);
3257 }
3258
3259 /* Expand atomic bit test and reset. */
3260
3261 static void
expand_ATOMIC_BIT_TEST_AND_RESET(internal_fn,gcall * call)3262 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3263 {
3264 expand_ifn_atomic_bit_test_and (call);
3265 }
3266
3267 /* Expand atomic bit test and set. */
3268
3269 static void
expand_ATOMIC_COMPARE_EXCHANGE(internal_fn,gcall * call)3270 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3271 {
3272 expand_ifn_atomic_compare_exchange (call);
3273 }
3274
3275 /* Expand atomic add fetch and cmp with 0. */
3276
3277 static void
expand_ATOMIC_ADD_FETCH_CMP_0(internal_fn,gcall * call)3278 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3279 {
3280 expand_ifn_atomic_op_fetch_cmp_0 (call);
3281 }
3282
3283 /* Expand atomic sub fetch and cmp with 0. */
3284
3285 static void
expand_ATOMIC_SUB_FETCH_CMP_0(internal_fn,gcall * call)3286 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3287 {
3288 expand_ifn_atomic_op_fetch_cmp_0 (call);
3289 }
3290
3291 /* Expand atomic and fetch and cmp with 0. */
3292
3293 static void
expand_ATOMIC_AND_FETCH_CMP_0(internal_fn,gcall * call)3294 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3295 {
3296 expand_ifn_atomic_op_fetch_cmp_0 (call);
3297 }
3298
3299 /* Expand atomic or fetch and cmp with 0. */
3300
3301 static void
expand_ATOMIC_OR_FETCH_CMP_0(internal_fn,gcall * call)3302 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3303 {
3304 expand_ifn_atomic_op_fetch_cmp_0 (call);
3305 }
3306
3307 /* Expand atomic xor fetch and cmp with 0. */
3308
3309 static void
expand_ATOMIC_XOR_FETCH_CMP_0(internal_fn,gcall * call)3310 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3311 {
3312 expand_ifn_atomic_op_fetch_cmp_0 (call);
3313 }
3314
3315 /* Expand LAUNDER to assignment, lhs = arg0. */
3316
3317 static void
expand_LAUNDER(internal_fn,gcall * call)3318 expand_LAUNDER (internal_fn, gcall *call)
3319 {
3320 tree lhs = gimple_call_lhs (call);
3321
3322 if (!lhs)
3323 return;
3324
3325 expand_assignment (lhs, gimple_call_arg (call, 0), false);
3326 }
3327
3328 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3329
3330 static void
expand_scatter_store_optab_fn(internal_fn,gcall * stmt,direct_optab optab)3331 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3332 {
3333 internal_fn ifn = gimple_call_internal_fn (stmt);
3334 int rhs_index = internal_fn_stored_value_index (ifn);
3335 int mask_index = internal_fn_mask_index (ifn);
3336 tree base = gimple_call_arg (stmt, 0);
3337 tree offset = gimple_call_arg (stmt, 1);
3338 tree scale = gimple_call_arg (stmt, 2);
3339 tree rhs = gimple_call_arg (stmt, rhs_index);
3340
3341 rtx base_rtx = expand_normal (base);
3342 rtx offset_rtx = expand_normal (offset);
3343 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3344 rtx rhs_rtx = expand_normal (rhs);
3345
3346 class expand_operand ops[6];
3347 int i = 0;
3348 create_address_operand (&ops[i++], base_rtx);
3349 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3350 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3351 create_integer_operand (&ops[i++], scale_int);
3352 create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3353 if (mask_index >= 0)
3354 {
3355 tree mask = gimple_call_arg (stmt, mask_index);
3356 rtx mask_rtx = expand_normal (mask);
3357 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
3358 }
3359
3360 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
3361 TYPE_MODE (TREE_TYPE (offset)));
3362 expand_insn (icode, i, ops);
3363 }
3364
3365 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3366
3367 static void
expand_gather_load_optab_fn(internal_fn,gcall * stmt,direct_optab optab)3368 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3369 {
3370 tree lhs = gimple_call_lhs (stmt);
3371 tree base = gimple_call_arg (stmt, 0);
3372 tree offset = gimple_call_arg (stmt, 1);
3373 tree scale = gimple_call_arg (stmt, 2);
3374
3375 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3376 rtx base_rtx = expand_normal (base);
3377 rtx offset_rtx = expand_normal (offset);
3378 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3379
3380 int i = 0;
3381 class expand_operand ops[6];
3382 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3383 create_address_operand (&ops[i++], base_rtx);
3384 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3385 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3386 create_integer_operand (&ops[i++], scale_int);
3387 if (optab == mask_gather_load_optab)
3388 {
3389 tree mask = gimple_call_arg (stmt, 4);
3390 rtx mask_rtx = expand_normal (mask);
3391 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
3392 }
3393 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
3394 TYPE_MODE (TREE_TYPE (offset)));
3395 expand_insn (icode, i, ops);
3396 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3397 emit_move_insn (lhs_rtx, ops[0].value);
3398 }
3399
3400 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3401 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3402
3403 static bool
contains_call_div_mod(rtx_insn * insn)3404 contains_call_div_mod (rtx_insn *insn)
3405 {
3406 subrtx_iterator::array_type array;
3407 for (; insn; insn = NEXT_INSN (insn))
3408 if (CALL_P (insn))
3409 return true;
3410 else if (INSN_P (insn))
3411 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3412 switch (GET_CODE (*iter))
3413 {
3414 case CALL:
3415 case DIV:
3416 case UDIV:
3417 case MOD:
3418 case UMOD:
3419 return true;
3420 default:
3421 break;
3422 }
3423 return false;
3424 }
3425
3426 /* Expand DIVMOD() using:
3427 a) optab handler for udivmod/sdivmod if it is available.
3428 b) If optab_handler doesn't exist, generate call to
3429 target-specific divmod libfunc. */
3430
3431 static void
expand_DIVMOD(internal_fn,gcall * call_stmt)3432 expand_DIVMOD (internal_fn, gcall *call_stmt)
3433 {
3434 tree lhs = gimple_call_lhs (call_stmt);
3435 tree arg0 = gimple_call_arg (call_stmt, 0);
3436 tree arg1 = gimple_call_arg (call_stmt, 1);
3437
3438 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3439 tree type = TREE_TYPE (TREE_TYPE (lhs));
3440 machine_mode mode = TYPE_MODE (type);
3441 bool unsignedp = TYPE_UNSIGNED (type);
3442 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3443
3444 rtx op0 = expand_normal (arg0);
3445 rtx op1 = expand_normal (arg1);
3446 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3447
3448 rtx quotient = NULL_RTX, remainder = NULL_RTX;
3449 rtx_insn *insns = NULL;
3450
3451 if (TREE_CODE (arg1) == INTEGER_CST)
3452 {
3453 /* For DIVMOD by integral constants, there could be efficient code
3454 expanded inline e.g. using shifts and plus/minus. Try to expand
3455 the division and modulo and if it emits any library calls or any
3456 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3457 divmod libcall. */
3458 scalar_int_mode int_mode;
3459 if (remainder == NULL_RTX
3460 && optimize
3461 && CONST_INT_P (op1)
3462 && !pow2p_hwi (INTVAL (op1))
3463 && is_int_mode (TYPE_MODE (type), &int_mode)
3464 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3465 && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
3466 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
3467 && optimize_insn_for_speed_p ())
3468 {
3469 rtx_insn *last = get_last_insn ();
3470 remainder = NULL_RTX;
3471 quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3472 TYPE_UNSIGNED (type));
3473 if (quotient != NULL_RTX)
3474 {
3475 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
3476 {
3477 rtx_insn *move = emit_move_insn (quotient, quotient);
3478 set_dst_reg_note (move, REG_EQUAL,
3479 gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3480 ? UDIV : DIV, int_mode,
3481 copy_rtx (op0), op1),
3482 quotient);
3483 move = emit_move_insn (remainder, remainder);
3484 set_dst_reg_note (move, REG_EQUAL,
3485 gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3486 ? UMOD : MOD, int_mode,
3487 copy_rtx (op0), op1),
3488 quotient);
3489 }
3490 }
3491 else
3492 delete_insns_since (last);
3493 }
3494
3495 if (remainder == NULL_RTX)
3496 {
3497 struct separate_ops ops;
3498 ops.code = TRUNC_DIV_EXPR;
3499 ops.type = type;
3500 ops.op0 = make_tree (ops.type, op0);
3501 ops.op1 = arg1;
3502 ops.op2 = NULL_TREE;
3503 ops.location = gimple_location (call_stmt);
3504 start_sequence ();
3505 quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3506 if (contains_call_div_mod (get_insns ()))
3507 quotient = NULL_RTX;
3508 else
3509 {
3510 ops.code = TRUNC_MOD_EXPR;
3511 remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3512 EXPAND_NORMAL);
3513 if (contains_call_div_mod (get_insns ()))
3514 remainder = NULL_RTX;
3515 }
3516 if (remainder)
3517 insns = get_insns ();
3518 end_sequence ();
3519 }
3520 }
3521
3522 if (remainder)
3523 emit_insn (insns);
3524
3525 /* Check if optab_handler exists for divmod_optab for given mode. */
3526 else if (optab_handler (tab, mode) != CODE_FOR_nothing)
3527 {
3528 quotient = gen_reg_rtx (mode);
3529 remainder = gen_reg_rtx (mode);
3530 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3531 }
3532
3533 /* Generate call to divmod libfunc if it exists. */
3534 else if (rtx libfunc = optab_libfunc (tab, mode))
3535 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3536 "ient, &remainder);
3537
3538 else
3539 gcc_unreachable ();
3540
3541 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3542 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3543 make_tree (TREE_TYPE (arg0), quotient),
3544 make_tree (TREE_TYPE (arg1), remainder)),
3545 target, VOIDmode, EXPAND_NORMAL);
3546 }
3547
3548 /* Expand a NOP. */
3549
3550 static void
expand_NOP(internal_fn,gcall *)3551 expand_NOP (internal_fn, gcall *)
3552 {
3553 /* Nothing. But it shouldn't really prevail. */
3554 }
3555
3556 /* Coroutines, all should have been processed at this stage. */
3557
3558 static void
expand_CO_FRAME(internal_fn,gcall *)3559 expand_CO_FRAME (internal_fn, gcall *)
3560 {
3561 gcc_unreachable ();
3562 }
3563
3564 static void
expand_CO_YIELD(internal_fn,gcall *)3565 expand_CO_YIELD (internal_fn, gcall *)
3566 {
3567 gcc_unreachable ();
3568 }
3569
3570 static void
expand_CO_SUSPN(internal_fn,gcall *)3571 expand_CO_SUSPN (internal_fn, gcall *)
3572 {
3573 gcc_unreachable ();
3574 }
3575
3576 static void
expand_CO_ACTOR(internal_fn,gcall *)3577 expand_CO_ACTOR (internal_fn, gcall *)
3578 {
3579 gcc_unreachable ();
3580 }
3581
3582 /* Expand a call to FN using the operands in STMT. FN has a single
3583 output operand and NARGS input operands. */
3584
3585 static void
expand_direct_optab_fn(internal_fn fn,gcall * stmt,direct_optab optab,unsigned int nargs)3586 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3587 unsigned int nargs)
3588 {
3589 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
3590
3591 tree_pair types = direct_internal_fn_types (fn, stmt);
3592 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
3593 gcc_assert (icode != CODE_FOR_nothing);
3594
3595 tree lhs = gimple_call_lhs (stmt);
3596 rtx lhs_rtx = NULL_RTX;
3597 if (lhs)
3598 lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3599
3600 /* Do not assign directly to a promoted subreg, since there is no
3601 guarantee that the instruction will leave the upper bits of the
3602 register in the state required by SUBREG_PROMOTED_SIGN. */
3603 rtx dest = lhs_rtx;
3604 if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
3605 dest = NULL_RTX;
3606
3607 create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
3608
3609 for (unsigned int i = 0; i < nargs; ++i)
3610 {
3611 tree rhs = gimple_call_arg (stmt, i);
3612 tree rhs_type = TREE_TYPE (rhs);
3613 rtx rhs_rtx = expand_normal (rhs);
3614 if (INTEGRAL_TYPE_P (rhs_type))
3615 create_convert_operand_from (&ops[i + 1], rhs_rtx,
3616 TYPE_MODE (rhs_type),
3617 TYPE_UNSIGNED (rhs_type));
3618 else
3619 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
3620 }
3621
3622 expand_insn (icode, nargs + 1, ops);
3623 if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
3624 {
3625 /* If the return value has an integral type, convert the instruction
3626 result to that type. This is useful for things that return an
3627 int regardless of the size of the input. If the instruction result
3628 is smaller than required, assume that it is signed.
3629
3630 If the return value has a nonintegral type, its mode must match
3631 the instruction result. */
3632 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
3633 {
3634 /* If this is a scalar in a register that is stored in a wider
3635 mode than the declared mode, compute the result into its
3636 declared mode and then convert to the wider mode. */
3637 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
3638 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
3639 convert_move (SUBREG_REG (lhs_rtx), tmp,
3640 SUBREG_PROMOTED_SIGN (lhs_rtx));
3641 }
3642 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
3643 emit_move_insn (lhs_rtx, ops[0].value);
3644 else
3645 {
3646 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
3647 convert_move (lhs_rtx, ops[0].value, 0);
3648 }
3649 }
3650 }
3651
3652 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3653
3654 static void
expand_while_optab_fn(internal_fn,gcall * stmt,convert_optab optab)3655 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3656 {
3657 expand_operand ops[3];
3658 tree rhs_type[2];
3659
3660 tree lhs = gimple_call_lhs (stmt);
3661 tree lhs_type = TREE_TYPE (lhs);
3662 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3663 create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
3664
3665 for (unsigned int i = 0; i < 2; ++i)
3666 {
3667 tree rhs = gimple_call_arg (stmt, i);
3668 rhs_type[i] = TREE_TYPE (rhs);
3669 rtx rhs_rtx = expand_normal (rhs);
3670 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
3671 }
3672
3673 insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
3674 TYPE_MODE (lhs_type));
3675
3676 expand_insn (icode, 3, ops);
3677 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3678 emit_move_insn (lhs_rtx, ops[0].value);
3679 }
3680
3681 /* Expanders for optabs that can use expand_direct_optab_fn. */
3682
3683 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3684 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3685
3686 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3687 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3688
3689 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3690 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3691
3692 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3693 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3694
3695 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3696 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3697
3698 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3699 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3700
3701 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3702 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3703
3704 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3705 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3706
3707 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3708 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3709
3710 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3711 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3712
3713 /* RETURN_TYPE and ARGS are a return type and argument list that are
3714 in principle compatible with FN (which satisfies direct_internal_fn_p).
3715 Return the types that should be used to determine whether the
3716 target supports FN. */
3717
3718 tree_pair
direct_internal_fn_types(internal_fn fn,tree return_type,tree * args)3719 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3720 {
3721 const direct_internal_fn_info &info = direct_internal_fn (fn);
3722 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3723 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3724 return tree_pair (type0, type1);
3725 }
3726
3727 /* CALL is a call whose return type and arguments are in principle
3728 compatible with FN (which satisfies direct_internal_fn_p). Return the
3729 types that should be used to determine whether the target supports FN. */
3730
3731 tree_pair
direct_internal_fn_types(internal_fn fn,gcall * call)3732 direct_internal_fn_types (internal_fn fn, gcall *call)
3733 {
3734 const direct_internal_fn_info &info = direct_internal_fn (fn);
3735 tree op0 = (info.type0 < 0
3736 ? gimple_call_lhs (call)
3737 : gimple_call_arg (call, info.type0));
3738 tree op1 = (info.type1 < 0
3739 ? gimple_call_lhs (call)
3740 : gimple_call_arg (call, info.type1));
3741 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
3742 }
3743
3744 /* Return true if OPTAB is supported for TYPES (whose modes should be
3745 the same) when the optimization type is OPT_TYPE. Used for simple
3746 direct optabs. */
3747
3748 static bool
direct_optab_supported_p(direct_optab optab,tree_pair types,optimization_type opt_type)3749 direct_optab_supported_p (direct_optab optab, tree_pair types,
3750 optimization_type opt_type)
3751 {
3752 machine_mode mode = TYPE_MODE (types.first);
3753 gcc_checking_assert (mode == TYPE_MODE (types.second));
3754 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
3755 }
3756
3757 /* Return true if OPTAB is supported for TYPES, where the first type
3758 is the destination and the second type is the source. Used for
3759 convert optabs. */
3760
3761 static bool
convert_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)3762 convert_optab_supported_p (convert_optab optab, tree_pair types,
3763 optimization_type opt_type)
3764 {
3765 return (convert_optab_handler (optab, TYPE_MODE (types.first),
3766 TYPE_MODE (types.second), opt_type)
3767 != CODE_FOR_nothing);
3768 }
3769
3770 /* Return true if load/store lanes optab OPTAB is supported for
3771 array type TYPES.first when the optimization type is OPT_TYPE. */
3772
3773 static bool
multi_vector_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)3774 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
3775 optimization_type opt_type)
3776 {
3777 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
3778 machine_mode imode = TYPE_MODE (types.first);
3779 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
3780 return (convert_optab_handler (optab, imode, vmode, opt_type)
3781 != CODE_FOR_nothing);
3782 }
3783
3784 #define direct_unary_optab_supported_p direct_optab_supported_p
3785 #define direct_binary_optab_supported_p direct_optab_supported_p
3786 #define direct_ternary_optab_supported_p direct_optab_supported_p
3787 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3788 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3789 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3790 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3791 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3792 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3793 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3794 #define direct_len_load_optab_supported_p direct_optab_supported_p
3795 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3796 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3797 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3798 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
3799 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
3800 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3801 #define direct_len_store_optab_supported_p direct_optab_supported_p
3802 #define direct_while_optab_supported_p convert_optab_supported_p
3803 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3804 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3805 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3806 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3807 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3808
3809 /* Return the optab used by internal function FN. */
3810
3811 static optab
direct_internal_fn_optab(internal_fn fn,tree_pair types)3812 direct_internal_fn_optab (internal_fn fn, tree_pair types)
3813 {
3814 switch (fn)
3815 {
3816 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3817 case IFN_##CODE: break;
3818 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3819 case IFN_##CODE: return OPTAB##_optab;
3820 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3821 UNSIGNED_OPTAB, TYPE) \
3822 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3823 ? UNSIGNED_OPTAB ## _optab \
3824 : SIGNED_OPTAB ## _optab);
3825 #include "internal-fn.def"
3826
3827 case IFN_LAST:
3828 break;
3829 }
3830 gcc_unreachable ();
3831 }
3832
3833 /* Return the optab used by internal function FN. */
3834
3835 static optab
direct_internal_fn_optab(internal_fn fn)3836 direct_internal_fn_optab (internal_fn fn)
3837 {
3838 switch (fn)
3839 {
3840 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3841 case IFN_##CODE: break;
3842 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3843 case IFN_##CODE: return OPTAB##_optab;
3844 #include "internal-fn.def"
3845
3846 case IFN_LAST:
3847 break;
3848 }
3849 gcc_unreachable ();
3850 }
3851
3852 /* Return true if FN is supported for the types in TYPES when the
3853 optimization type is OPT_TYPE. The types are those associated with
3854 the "type0" and "type1" fields of FN's direct_internal_fn_info
3855 structure. */
3856
3857 bool
direct_internal_fn_supported_p(internal_fn fn,tree_pair types,optimization_type opt_type)3858 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
3859 optimization_type opt_type)
3860 {
3861 switch (fn)
3862 {
3863 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3864 case IFN_##CODE: break;
3865 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3866 case IFN_##CODE: \
3867 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3868 opt_type);
3869 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3870 UNSIGNED_OPTAB, TYPE) \
3871 case IFN_##CODE: \
3872 { \
3873 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3874 ? UNSIGNED_OPTAB ## _optab \
3875 : SIGNED_OPTAB ## _optab); \
3876 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3877 opt_type); \
3878 }
3879 #include "internal-fn.def"
3880
3881 case IFN_LAST:
3882 break;
3883 }
3884 gcc_unreachable ();
3885 }
3886
3887 /* Return true if FN is supported for type TYPE when the optimization
3888 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3889 fields of FN's direct_internal_fn_info structure are the same. */
3890
3891 bool
direct_internal_fn_supported_p(internal_fn fn,tree type,optimization_type opt_type)3892 direct_internal_fn_supported_p (internal_fn fn, tree type,
3893 optimization_type opt_type)
3894 {
3895 const direct_internal_fn_info &info = direct_internal_fn (fn);
3896 gcc_checking_assert (info.type0 == info.type1);
3897 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
3898 }
3899
3900 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3901 given that STMT is a call to a direct internal function. */
3902
3903 bool
direct_internal_fn_supported_p(gcall * stmt,optimization_type opt_type)3904 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
3905 {
3906 internal_fn fn = gimple_call_internal_fn (stmt);
3907 tree_pair types = direct_internal_fn_types (fn, stmt);
3908 return direct_internal_fn_supported_p (fn, types, opt_type);
3909 }
3910
3911 /* Return true if FN is a binary operation and if FN is commutative. */
3912
3913 bool
commutative_binary_fn_p(internal_fn fn)3914 commutative_binary_fn_p (internal_fn fn)
3915 {
3916 switch (fn)
3917 {
3918 case IFN_AVG_FLOOR:
3919 case IFN_AVG_CEIL:
3920 case IFN_MULH:
3921 case IFN_MULHS:
3922 case IFN_MULHRS:
3923 case IFN_FMIN:
3924 case IFN_FMAX:
3925 case IFN_COMPLEX_MUL:
3926 case IFN_UBSAN_CHECK_ADD:
3927 case IFN_UBSAN_CHECK_MUL:
3928 case IFN_ADD_OVERFLOW:
3929 case IFN_MUL_OVERFLOW:
3930 return true;
3931
3932 default:
3933 return false;
3934 }
3935 }
3936
3937 /* Return true if FN is a ternary operation and if its first two arguments
3938 are commutative. */
3939
3940 bool
commutative_ternary_fn_p(internal_fn fn)3941 commutative_ternary_fn_p (internal_fn fn)
3942 {
3943 switch (fn)
3944 {
3945 case IFN_FMA:
3946 case IFN_FMS:
3947 case IFN_FNMA:
3948 case IFN_FNMS:
3949 return true;
3950
3951 default:
3952 return false;
3953 }
3954 }
3955
3956 /* Return true if FN is an associative binary operation. */
3957
3958 bool
associative_binary_fn_p(internal_fn fn)3959 associative_binary_fn_p (internal_fn fn)
3960 {
3961 switch (fn)
3962 {
3963 case IFN_FMIN:
3964 case IFN_FMAX:
3965 return true;
3966
3967 default:
3968 return false;
3969 }
3970 }
3971
3972 /* If FN is commutative in two consecutive arguments, return the
3973 index of the first, otherwise return -1. */
3974
3975 int
first_commutative_argument(internal_fn fn)3976 first_commutative_argument (internal_fn fn)
3977 {
3978 switch (fn)
3979 {
3980 case IFN_COND_ADD:
3981 case IFN_COND_MUL:
3982 case IFN_COND_MIN:
3983 case IFN_COND_MAX:
3984 case IFN_COND_FMIN:
3985 case IFN_COND_FMAX:
3986 case IFN_COND_AND:
3987 case IFN_COND_IOR:
3988 case IFN_COND_XOR:
3989 case IFN_COND_FMA:
3990 case IFN_COND_FMS:
3991 case IFN_COND_FNMA:
3992 case IFN_COND_FNMS:
3993 return 1;
3994
3995 default:
3996 if (commutative_binary_fn_p (fn)
3997 || commutative_ternary_fn_p (fn))
3998 return 0;
3999 return -1;
4000 }
4001 }
4002
4003 /* Return true if IFN_SET_EDOM is supported. */
4004
4005 bool
set_edom_supported_p(void)4006 set_edom_supported_p (void)
4007 {
4008 #ifdef TARGET_EDOM
4009 return true;
4010 #else
4011 return false;
4012 #endif
4013 }
4014
4015 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4016 static void \
4017 expand_##CODE (internal_fn fn, gcall *stmt) \
4018 { \
4019 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4020 }
4021 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4022 UNSIGNED_OPTAB, TYPE) \
4023 static void \
4024 expand_##CODE (internal_fn fn, gcall *stmt) \
4025 { \
4026 tree_pair types = direct_internal_fn_types (fn, stmt); \
4027 optab which_optab = direct_internal_fn_optab (fn, types); \
4028 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4029 }
4030 #include "internal-fn.def"
4031
4032 /* Routines to expand each internal function, indexed by function number.
4033 Each routine has the prototype:
4034
4035 expand_<NAME> (gcall *stmt)
4036
4037 where STMT is the statement that performs the call. */
4038 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4039 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4040 #include "internal-fn.def"
4041 0
4042 };
4043
4044 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
4045 tree code CODE. */
4046 #define FOR_EACH_CODE_MAPPING(T) \
4047 T (PLUS_EXPR, IFN_COND_ADD) \
4048 T (MINUS_EXPR, IFN_COND_SUB) \
4049 T (MULT_EXPR, IFN_COND_MUL) \
4050 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
4051 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
4052 T (RDIV_EXPR, IFN_COND_RDIV) \
4053 T (MIN_EXPR, IFN_COND_MIN) \
4054 T (MAX_EXPR, IFN_COND_MAX) \
4055 T (BIT_AND_EXPR, IFN_COND_AND) \
4056 T (BIT_IOR_EXPR, IFN_COND_IOR) \
4057 T (BIT_XOR_EXPR, IFN_COND_XOR) \
4058 T (LSHIFT_EXPR, IFN_COND_SHL) \
4059 T (RSHIFT_EXPR, IFN_COND_SHR) \
4060 T (NEGATE_EXPR, IFN_COND_NEG)
4061
4062 /* Return a function that only performs CODE when a certain condition is met
4063 and that uses a given fallback value otherwise. For example, if CODE is
4064 a binary operation associated with conditional function FN:
4065
4066 LHS = FN (COND, A, B, ELSE)
4067
4068 is equivalent to the C expression:
4069
4070 LHS = COND ? A CODE B : ELSE;
4071
4072 operating elementwise if the operands are vectors.
4073
4074 Return IFN_LAST if no such function exists. */
4075
4076 internal_fn
get_conditional_internal_fn(tree_code code)4077 get_conditional_internal_fn (tree_code code)
4078 {
4079 switch (code)
4080 {
4081 #define CASE(CODE, IFN) case CODE: return IFN;
4082 FOR_EACH_CODE_MAPPING(CASE)
4083 #undef CASE
4084 default:
4085 return IFN_LAST;
4086 }
4087 }
4088
4089 /* If IFN implements the conditional form of a tree code, return that
4090 tree code, otherwise return ERROR_MARK. */
4091
4092 tree_code
conditional_internal_fn_code(internal_fn ifn)4093 conditional_internal_fn_code (internal_fn ifn)
4094 {
4095 switch (ifn)
4096 {
4097 #define CASE(CODE, IFN) case IFN: return CODE;
4098 FOR_EACH_CODE_MAPPING(CASE)
4099 #undef CASE
4100 default:
4101 return ERROR_MARK;
4102 }
4103 }
4104
4105 /* Invoke T(IFN) for each internal function IFN that also has an
4106 IFN_COND_* form. */
4107 #define FOR_EACH_COND_FN_PAIR(T) \
4108 T (FMAX) \
4109 T (FMIN) \
4110 T (FMA) \
4111 T (FMS) \
4112 T (FNMA) \
4113 T (FNMS)
4114
4115 /* Return a function that only performs internal function FN when a
4116 certain condition is met and that uses a given fallback value otherwise.
4117 In other words, the returned function FN' is such that:
4118
4119 LHS = FN' (COND, A1, ... An, ELSE)
4120
4121 is equivalent to the C expression:
4122
4123 LHS = COND ? FN (A1, ..., An) : ELSE;
4124
4125 operating elementwise if the operands are vectors.
4126
4127 Return IFN_LAST if no such function exists. */
4128
4129 internal_fn
get_conditional_internal_fn(internal_fn fn)4130 get_conditional_internal_fn (internal_fn fn)
4131 {
4132 switch (fn)
4133 {
4134 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4135 FOR_EACH_COND_FN_PAIR(CASE)
4136 #undef CASE
4137 default:
4138 return IFN_LAST;
4139 }
4140 }
4141
4142 /* If IFN implements the conditional form of an unconditional internal
4143 function, return that unconditional function, otherwise return IFN_LAST. */
4144
4145 internal_fn
get_unconditional_internal_fn(internal_fn ifn)4146 get_unconditional_internal_fn (internal_fn ifn)
4147 {
4148 switch (ifn)
4149 {
4150 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
4151 FOR_EACH_COND_FN_PAIR(CASE)
4152 #undef CASE
4153 default:
4154 return IFN_LAST;
4155 }
4156 }
4157
4158 /* Return true if STMT can be interpreted as a conditional tree code
4159 operation of the form:
4160
4161 LHS = COND ? OP (RHS1, ...) : ELSE;
4162
4163 operating elementwise if the operands are vectors. This includes
4164 the case of an all-true COND, so that the operation always happens.
4165
4166 When returning true, set:
4167
4168 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4169 is known to be all-true
4170 - *CODE_OUT to the tree code
4171 - OPS[I] to operand I of *CODE_OUT
4172 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4173 condition is known to be all true. */
4174
4175 bool
can_interpret_as_conditional_op_p(gimple * stmt,tree * cond_out,tree_code * code_out,tree (& ops)[3],tree * else_out)4176 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4177 tree_code *code_out,
4178 tree (&ops)[3], tree *else_out)
4179 {
4180 if (gassign *assign = dyn_cast <gassign *> (stmt))
4181 {
4182 *cond_out = NULL_TREE;
4183 *code_out = gimple_assign_rhs_code (assign);
4184 ops[0] = gimple_assign_rhs1 (assign);
4185 ops[1] = gimple_assign_rhs2 (assign);
4186 ops[2] = gimple_assign_rhs3 (assign);
4187 *else_out = NULL_TREE;
4188 return true;
4189 }
4190 if (gcall *call = dyn_cast <gcall *> (stmt))
4191 if (gimple_call_internal_p (call))
4192 {
4193 internal_fn ifn = gimple_call_internal_fn (call);
4194 tree_code code = conditional_internal_fn_code (ifn);
4195 if (code != ERROR_MARK)
4196 {
4197 *cond_out = gimple_call_arg (call, 0);
4198 *code_out = code;
4199 unsigned int nops = gimple_call_num_args (call) - 2;
4200 for (unsigned int i = 0; i < 3; ++i)
4201 ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
4202 *else_out = gimple_call_arg (call, nops + 1);
4203 if (integer_truep (*cond_out))
4204 {
4205 *cond_out = NULL_TREE;
4206 *else_out = NULL_TREE;
4207 }
4208 return true;
4209 }
4210 }
4211 return false;
4212 }
4213
4214 /* Return true if IFN is some form of load from memory. */
4215
4216 bool
internal_load_fn_p(internal_fn fn)4217 internal_load_fn_p (internal_fn fn)
4218 {
4219 switch (fn)
4220 {
4221 case IFN_MASK_LOAD:
4222 case IFN_LOAD_LANES:
4223 case IFN_MASK_LOAD_LANES:
4224 case IFN_GATHER_LOAD:
4225 case IFN_MASK_GATHER_LOAD:
4226 case IFN_LEN_LOAD:
4227 return true;
4228
4229 default:
4230 return false;
4231 }
4232 }
4233
4234 /* Return true if IFN is some form of store to memory. */
4235
4236 bool
internal_store_fn_p(internal_fn fn)4237 internal_store_fn_p (internal_fn fn)
4238 {
4239 switch (fn)
4240 {
4241 case IFN_MASK_STORE:
4242 case IFN_STORE_LANES:
4243 case IFN_MASK_STORE_LANES:
4244 case IFN_SCATTER_STORE:
4245 case IFN_MASK_SCATTER_STORE:
4246 case IFN_LEN_STORE:
4247 return true;
4248
4249 default:
4250 return false;
4251 }
4252 }
4253
4254 /* Return true if IFN is some form of gather load or scatter store. */
4255
4256 bool
internal_gather_scatter_fn_p(internal_fn fn)4257 internal_gather_scatter_fn_p (internal_fn fn)
4258 {
4259 switch (fn)
4260 {
4261 case IFN_GATHER_LOAD:
4262 case IFN_MASK_GATHER_LOAD:
4263 case IFN_SCATTER_STORE:
4264 case IFN_MASK_SCATTER_STORE:
4265 return true;
4266
4267 default:
4268 return false;
4269 }
4270 }
4271
4272 /* If FN takes a vector mask argument, return the index of that argument,
4273 otherwise return -1. */
4274
4275 int
internal_fn_mask_index(internal_fn fn)4276 internal_fn_mask_index (internal_fn fn)
4277 {
4278 switch (fn)
4279 {
4280 case IFN_MASK_LOAD:
4281 case IFN_MASK_LOAD_LANES:
4282 case IFN_MASK_STORE:
4283 case IFN_MASK_STORE_LANES:
4284 return 2;
4285
4286 case IFN_MASK_GATHER_LOAD:
4287 case IFN_MASK_SCATTER_STORE:
4288 return 4;
4289
4290 default:
4291 return (conditional_internal_fn_code (fn) != ERROR_MARK
4292 || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
4293 }
4294 }
4295
4296 /* If FN takes a value that should be stored to memory, return the index
4297 of that argument, otherwise return -1. */
4298
4299 int
internal_fn_stored_value_index(internal_fn fn)4300 internal_fn_stored_value_index (internal_fn fn)
4301 {
4302 switch (fn)
4303 {
4304 case IFN_MASK_STORE:
4305 case IFN_MASK_STORE_LANES:
4306 case IFN_SCATTER_STORE:
4307 case IFN_MASK_SCATTER_STORE:
4308 case IFN_LEN_STORE:
4309 return 3;
4310
4311 default:
4312 return -1;
4313 }
4314 }
4315
4316 /* Return true if the target supports gather load or scatter store function
4317 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4318 while for stores it is the vector type of the stored data argument.
4319 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4320 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4321 offset from the shared base address of each loaded or stored element.
4322 SCALE is the amount by which these offsets should be multiplied
4323 *after* they have been extended to address width. */
4324
4325 bool
internal_gather_scatter_fn_supported_p(internal_fn ifn,tree vector_type,tree memory_element_type,tree offset_vector_type,int scale)4326 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
4327 tree memory_element_type,
4328 tree offset_vector_type, int scale)
4329 {
4330 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
4331 TYPE_SIZE (memory_element_type)))
4332 return false;
4333 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
4334 TYPE_VECTOR_SUBPARTS (offset_vector_type)))
4335 return false;
4336 optab optab = direct_internal_fn_optab (ifn);
4337 insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
4338 TYPE_MODE (offset_vector_type));
4339 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
4340 bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
4341 return (icode != CODE_FOR_nothing
4342 && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
4343 && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale)));
4344 }
4345
4346 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4347 for pointers of type TYPE when the accesses have LENGTH bytes and their
4348 common byte alignment is ALIGN. */
4349
4350 bool
internal_check_ptrs_fn_supported_p(internal_fn ifn,tree type,poly_uint64 length,unsigned int align)4351 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
4352 poly_uint64 length, unsigned int align)
4353 {
4354 machine_mode mode = TYPE_MODE (type);
4355 optab optab = direct_internal_fn_optab (ifn);
4356 insn_code icode = direct_optab_handler (optab, mode);
4357 if (icode == CODE_FOR_nothing)
4358 return false;
4359 rtx length_rtx = immed_wide_int_const (length, mode);
4360 return (insn_operand_matches (icode, 3, length_rtx)
4361 && insn_operand_matches (icode, 4, GEN_INT (align)));
4362 }
4363
4364 /* Return the supported bias for IFN which is either IFN_LEN_LOAD
4365 or IFN_LEN_STORE. For now we only support the biases of 0 and -1
4366 (in case 0 is not an allowable length for len_load or len_store).
4367 If none of the biases match what the backend provides, return
4368 VECT_PARTIAL_BIAS_UNSUPPORTED. */
4369
4370 signed char
internal_len_load_store_bias(internal_fn ifn,machine_mode mode)4371 internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
4372 {
4373 optab optab = direct_internal_fn_optab (ifn);
4374 insn_code icode = direct_optab_handler (optab, mode);
4375
4376 if (icode != CODE_FOR_nothing)
4377 {
4378 /* For now we only support biases of 0 or -1. Try both of them. */
4379 if (insn_operand_matches (icode, 3, GEN_INT (0)))
4380 return 0;
4381 if (insn_operand_matches (icode, 3, GEN_INT (-1)))
4382 return -1;
4383 }
4384
4385 return VECT_PARTIAL_BIAS_UNSUPPORTED;
4386 }
4387
4388 /* Expand STMT as though it were a call to internal function FN. */
4389
4390 void
expand_internal_call(internal_fn fn,gcall * stmt)4391 expand_internal_call (internal_fn fn, gcall *stmt)
4392 {
4393 internal_fn_expanders[fn] (fn, stmt);
4394 }
4395
4396 /* Expand STMT, which is a call to internal function FN. */
4397
4398 void
expand_internal_call(gcall * stmt)4399 expand_internal_call (gcall *stmt)
4400 {
4401 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
4402 }
4403
4404 /* If TYPE is a vector type, return true if IFN is a direct internal
4405 function that is supported for that type. If TYPE is a scalar type,
4406 return true if IFN is a direct internal function that is supported for
4407 the target's preferred vector version of TYPE. */
4408
4409 bool
vectorized_internal_fn_supported_p(internal_fn ifn,tree type)4410 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
4411 {
4412 if (VECTOR_MODE_P (TYPE_MODE (type)))
4413 return direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED);
4414
4415 scalar_mode smode;
4416 if (VECTOR_TYPE_P (type)
4417 || !is_a <scalar_mode> (TYPE_MODE (type), &smode))
4418 return false;
4419
4420 machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
4421 if (VECTOR_MODE_P (vmode))
4422 {
4423 tree vectype = build_vector_type_for_mode (type, vmode);
4424 if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4425 return true;
4426 }
4427
4428 auto_vector_modes vector_modes;
4429 targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
4430 for (machine_mode base_mode : vector_modes)
4431 if (related_vector_mode (base_mode, smode).exists (&vmode))
4432 {
4433 tree vectype = build_vector_type_for_mode (type, vmode);
4434 if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4435 return true;
4436 }
4437
4438 return false;
4439 }
4440
4441 void
expand_SHUFFLEVECTOR(internal_fn,gcall *)4442 expand_SHUFFLEVECTOR (internal_fn, gcall *)
4443 {
4444 gcc_unreachable ();
4445 }
4446
4447 void
expand_PHI(internal_fn,gcall *)4448 expand_PHI (internal_fn, gcall *)
4449 {
4450 gcc_unreachable ();
4451 }
4452
4453 void
expand_SPACESHIP(internal_fn,gcall * stmt)4454 expand_SPACESHIP (internal_fn, gcall *stmt)
4455 {
4456 tree lhs = gimple_call_lhs (stmt);
4457 tree rhs1 = gimple_call_arg (stmt, 0);
4458 tree rhs2 = gimple_call_arg (stmt, 1);
4459 tree type = TREE_TYPE (rhs1);
4460
4461 do_pending_stack_adjust ();
4462
4463 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4464 rtx op1 = expand_normal (rhs1);
4465 rtx op2 = expand_normal (rhs2);
4466
4467 class expand_operand ops[3];
4468 create_output_operand (&ops[0], target, TYPE_MODE (TREE_TYPE (lhs)));
4469 create_input_operand (&ops[1], op1, TYPE_MODE (type));
4470 create_input_operand (&ops[2], op2, TYPE_MODE (type));
4471 insn_code icode = optab_handler (spaceship_optab, TYPE_MODE (type));
4472 expand_insn (icode, 3, ops);
4473 if (!rtx_equal_p (target, ops[0].value))
4474 emit_move_insn (target, ops[0].value);
4475 }
4476