1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2022 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "rtl.h"
27 #include "tree.h"
28 #include "memmodel.h"
29 #include "predict.h"
30 #include "tm_p.h"
31 #include "optabs.h"
32 #include "expmed.h"
33 #include "emit-rtl.h"
34 #include "recog.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
37
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
41 #include "except.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "expr.h"
45 #include "optabs-tree.h"
46 #include "libfuncs.h"
47 #include "internal-fn.h"
48 #include "langhooks.h"
49
50 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
51 machine_mode *);
52 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
53 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
54
55 static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
56
57 /* Debug facility for use in GDB. */
58 void debug_optab_libfuncs (void);
59
60 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
61 the result of operation CODE applied to OP0 (and OP1 if it is a binary
62 operation). OP0_MODE is OP0's mode.
63
64 If the last insn does not set TARGET, don't do anything, but return 1.
65
66 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
67 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
68 try again, ensuring that TARGET is not one of the operands. */
69
70 static int
add_equal_note(rtx_insn * insns,rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode op0_mode)71 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
72 rtx op1, machine_mode op0_mode)
73 {
74 rtx_insn *last_insn;
75 rtx set;
76 rtx note;
77
78 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
79
80 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
81 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
82 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
83 && GET_RTX_CLASS (code) != RTX_COMPARE
84 && GET_RTX_CLASS (code) != RTX_UNARY)
85 return 1;
86
87 if (GET_CODE (target) == ZERO_EXTRACT)
88 return 1;
89
90 for (last_insn = insns;
91 NEXT_INSN (last_insn) != NULL_RTX;
92 last_insn = NEXT_INSN (last_insn))
93 ;
94
95 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
96 a value changing in the insn, so the note would be invalid for CSE. */
97 if (reg_overlap_mentioned_p (target, op0)
98 || (op1 && reg_overlap_mentioned_p (target, op1)))
99 {
100 if (MEM_P (target)
101 && (rtx_equal_p (target, op0)
102 || (op1 && rtx_equal_p (target, op1))))
103 {
104 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
105 over expanding it as temp = MEM op X, MEM = temp. If the target
106 supports MEM = MEM op X instructions, it is sometimes too hard
107 to reconstruct that form later, especially if X is also a memory,
108 and due to multiple occurrences of addresses the address might
109 be forced into register unnecessarily.
110 Note that not emitting the REG_EQUIV note might inhibit
111 CSE in some cases. */
112 set = single_set (last_insn);
113 if (set
114 && GET_CODE (SET_SRC (set)) == code
115 && MEM_P (SET_DEST (set))
116 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
117 || (op1 && rtx_equal_p (SET_DEST (set),
118 XEXP (SET_SRC (set), 1)))))
119 return 1;
120 }
121 return 0;
122 }
123
124 set = set_for_reg_notes (last_insn);
125 if (set == NULL_RTX)
126 return 1;
127
128 if (! rtx_equal_p (SET_DEST (set), target)
129 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
130 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
131 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
132 return 1;
133
134 if (GET_RTX_CLASS (code) == RTX_UNARY)
135 switch (code)
136 {
137 case FFS:
138 case CLZ:
139 case CTZ:
140 case CLRSB:
141 case POPCOUNT:
142 case PARITY:
143 case BSWAP:
144 if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
145 {
146 note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
147 if (GET_MODE_UNIT_SIZE (op0_mode)
148 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
149 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
150 note, op0_mode);
151 else
152 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
153 note, op0_mode);
154 break;
155 }
156 /* FALLTHRU */
157 default:
158 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
159 break;
160 }
161 else
162 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
163
164 set_unique_reg_note (last_insn, REG_EQUAL, note);
165
166 return 1;
167 }
168
169 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
170 for a widening operation would be. In most cases this would be OP0, but if
171 that's a constant it'll be VOIDmode, which isn't useful. */
172
173 static machine_mode
widened_mode(machine_mode to_mode,rtx op0,rtx op1)174 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
175 {
176 machine_mode m0 = GET_MODE (op0);
177 machine_mode m1 = GET_MODE (op1);
178 machine_mode result;
179
180 if (m0 == VOIDmode && m1 == VOIDmode)
181 return to_mode;
182 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
183 result = m1;
184 else
185 result = m0;
186
187 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
188 return to_mode;
189
190 return result;
191 }
192
193 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
194 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
195 not actually do a sign-extend or zero-extend, but can leave the
196 higher-order bits of the result rtx undefined, for example, in the case
197 of logical operations, but not right shifts. */
198
199 static rtx
widen_operand(rtx op,machine_mode mode,machine_mode oldmode,int unsignedp,int no_extend)200 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
201 int unsignedp, int no_extend)
202 {
203 rtx result;
204 scalar_int_mode int_mode;
205
206 /* If we don't have to extend and this is a constant, return it. */
207 if (no_extend && GET_MODE (op) == VOIDmode)
208 return op;
209
210 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
211 extend since it will be more efficient to do so unless the signedness of
212 a promoted object differs from our extension. */
213 if (! no_extend
214 || !is_a <scalar_int_mode> (mode, &int_mode)
215 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
216 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
217 return convert_modes (mode, oldmode, op, unsignedp);
218
219 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
220 SUBREG. */
221 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
222 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
223
224 /* Otherwise, get an object of MODE, clobber it, and set the low-order
225 part to OP. */
226
227 result = gen_reg_rtx (int_mode);
228 emit_clobber (result);
229 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
230 return result;
231 }
232
233 /* Expand vector widening operations.
234
235 There are two different classes of operations handled here:
236 1) Operations whose result is wider than all the arguments to the operation.
237 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
238 In this case OP0 and optionally OP1 would be initialized,
239 but WIDE_OP wouldn't (not relevant for this case).
240 2) Operations whose result is of the same size as the last argument to the
241 operation, but wider than all the other arguments to the operation.
242 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
243 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
244
245 E.g, when called to expand the following operations, this is how
246 the arguments will be initialized:
247 nops OP0 OP1 WIDE_OP
248 widening-sum 2 oprnd0 - oprnd1
249 widening-dot-product 3 oprnd0 oprnd1 oprnd2
250 widening-mult 2 oprnd0 oprnd1 -
251 type-promotion (vec-unpack) 1 oprnd0 - - */
252
253 rtx
expand_widen_pattern_expr(sepops ops,rtx op0,rtx op1,rtx wide_op,rtx target,int unsignedp)254 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
255 rtx target, int unsignedp)
256 {
257 class expand_operand eops[4];
258 tree oprnd0, oprnd1, oprnd2;
259 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
260 optab widen_pattern_optab;
261 enum insn_code icode;
262 int nops = TREE_CODE_LENGTH (ops->code);
263 int op;
264 bool sbool = false;
265
266 oprnd0 = ops->op0;
267 if (nops >= 2)
268 oprnd1 = ops->op1;
269 if (nops >= 3)
270 oprnd2 = ops->op2;
271
272 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
273 if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
274 || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
275 /* The sign is from the result type rather than operand's type
276 for these ops. */
277 widen_pattern_optab
278 = optab_for_tree_code (ops->code, ops->type, optab_default);
279 else if ((ops->code == VEC_UNPACK_HI_EXPR
280 || ops->code == VEC_UNPACK_LO_EXPR)
281 && VECTOR_BOOLEAN_TYPE_P (ops->type)
282 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
283 && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
284 && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
285 {
286 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
287 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
288 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
289 the pattern number of elements in the wider vector. */
290 widen_pattern_optab
291 = (ops->code == VEC_UNPACK_HI_EXPR
292 ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
293 sbool = true;
294 }
295 else if (ops->code == DOT_PROD_EXPR)
296 {
297 enum optab_subtype subtype = optab_default;
298 signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
299 signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
300 if (sign1 == sign2)
301 ;
302 else if (sign1 == SIGNED && sign2 == UNSIGNED)
303 {
304 subtype = optab_vector_mixed_sign;
305 /* Same as optab_vector_mixed_sign but flip the operands. */
306 std::swap (op0, op1);
307 }
308 else if (sign1 == UNSIGNED && sign2 == SIGNED)
309 subtype = optab_vector_mixed_sign;
310 else
311 gcc_unreachable ();
312
313 widen_pattern_optab
314 = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
315 }
316 else
317 widen_pattern_optab
318 = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
319 if (ops->code == WIDEN_MULT_PLUS_EXPR
320 || ops->code == WIDEN_MULT_MINUS_EXPR)
321 icode = find_widening_optab_handler (widen_pattern_optab,
322 TYPE_MODE (TREE_TYPE (ops->op2)),
323 tmode0);
324 else
325 icode = optab_handler (widen_pattern_optab, tmode0);
326 gcc_assert (icode != CODE_FOR_nothing);
327
328 if (nops >= 2)
329 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
330 else if (sbool)
331 {
332 nops = 2;
333 op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
334 tmode1 = tmode0;
335 }
336
337 /* The last operand is of a wider mode than the rest of the operands. */
338 if (nops == 2)
339 wmode = tmode1;
340 else if (nops == 3)
341 {
342 gcc_assert (tmode1 == tmode0);
343 gcc_assert (op1);
344 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
345 }
346
347 op = 0;
348 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
349 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
350 if (op1)
351 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
352 if (wide_op)
353 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
354 expand_insn (icode, op, eops);
355 return eops[0].value;
356 }
357
358 /* Generate code to perform an operation specified by TERNARY_OPTAB
359 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
360
361 UNSIGNEDP is for the case where we have to widen the operands
362 to perform the operation. It says to use zero-extension.
363
364 If TARGET is nonzero, the value
365 is generated there, if it is convenient to do so.
366 In all cases an rtx is returned for the locus of the value;
367 this may or may not be TARGET. */
368
369 rtx
expand_ternary_op(machine_mode mode,optab ternary_optab,rtx op0,rtx op1,rtx op2,rtx target,int unsignedp)370 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
371 rtx op1, rtx op2, rtx target, int unsignedp)
372 {
373 class expand_operand ops[4];
374 enum insn_code icode = optab_handler (ternary_optab, mode);
375
376 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
377
378 create_output_operand (&ops[0], target, mode);
379 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
380 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
381 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
382 expand_insn (icode, 4, ops);
383 return ops[0].value;
384 }
385
386
387 /* Like expand_binop, but return a constant rtx if the result can be
388 calculated at compile time. The arguments and return value are
389 otherwise the same as for expand_binop. */
390
391 rtx
simplify_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)392 simplify_expand_binop (machine_mode mode, optab binoptab,
393 rtx op0, rtx op1, rtx target, int unsignedp,
394 enum optab_methods methods)
395 {
396 if (CONSTANT_P (op0) && CONSTANT_P (op1))
397 {
398 rtx x = simplify_binary_operation (optab_to_code (binoptab),
399 mode, op0, op1);
400 if (x)
401 return x;
402 }
403
404 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
405 }
406
407 /* Like simplify_expand_binop, but always put the result in TARGET.
408 Return true if the expansion succeeded. */
409
410 bool
force_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)411 force_expand_binop (machine_mode mode, optab binoptab,
412 rtx op0, rtx op1, rtx target, int unsignedp,
413 enum optab_methods methods)
414 {
415 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
416 target, unsignedp, methods);
417 if (x == 0)
418 return false;
419 if (x != target)
420 emit_move_insn (target, x);
421 return true;
422 }
423
424 /* Create a new vector value in VMODE with all elements set to OP. The
425 mode of OP must be the element mode of VMODE. If OP is a constant,
426 then the return value will be a constant. */
427
428 rtx
expand_vector_broadcast(machine_mode vmode,rtx op)429 expand_vector_broadcast (machine_mode vmode, rtx op)
430 {
431 int n;
432 rtvec vec;
433
434 gcc_checking_assert (VECTOR_MODE_P (vmode));
435
436 if (valid_for_const_vector_p (vmode, op))
437 return gen_const_vec_duplicate (vmode, op);
438
439 insn_code icode = optab_handler (vec_duplicate_optab, vmode);
440 if (icode != CODE_FOR_nothing)
441 {
442 class expand_operand ops[2];
443 create_output_operand (&ops[0], NULL_RTX, vmode);
444 create_input_operand (&ops[1], op, GET_MODE (op));
445 expand_insn (icode, 2, ops);
446 return ops[0].value;
447 }
448
449 if (!GET_MODE_NUNITS (vmode).is_constant (&n))
450 return NULL;
451
452 /* ??? If the target doesn't have a vec_init, then we have no easy way
453 of performing this operation. Most of this sort of generic support
454 is hidden away in the vector lowering support in gimple. */
455 icode = convert_optab_handler (vec_init_optab, vmode,
456 GET_MODE_INNER (vmode));
457 if (icode == CODE_FOR_nothing)
458 return NULL;
459
460 vec = rtvec_alloc (n);
461 for (int i = 0; i < n; ++i)
462 RTVEC_ELT (vec, i) = op;
463 rtx ret = gen_reg_rtx (vmode);
464 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
465
466 return ret;
467 }
468
469 /* This subroutine of expand_doubleword_shift handles the cases in which
470 the effective shift value is >= BITS_PER_WORD. The arguments and return
471 value are the same as for the parent routine, except that SUPERWORD_OP1
472 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
473 INTO_TARGET may be null if the caller has decided to calculate it. */
474
475 static bool
expand_superword_shift(optab binoptab,rtx outof_input,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods)476 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
477 rtx outof_target, rtx into_target,
478 int unsignedp, enum optab_methods methods)
479 {
480 if (into_target != 0)
481 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
482 into_target, unsignedp, methods))
483 return false;
484
485 if (outof_target != 0)
486 {
487 /* For a signed right shift, we must fill OUTOF_TARGET with copies
488 of the sign bit, otherwise we must fill it with zeros. */
489 if (binoptab != ashr_optab)
490 emit_move_insn (outof_target, CONST0_RTX (word_mode));
491 else
492 if (!force_expand_binop (word_mode, binoptab, outof_input,
493 gen_int_shift_amount (word_mode,
494 BITS_PER_WORD - 1),
495 outof_target, unsignedp, methods))
496 return false;
497 }
498 return true;
499 }
500
501 /* This subroutine of expand_doubleword_shift handles the cases in which
502 the effective shift value is < BITS_PER_WORD. The arguments and return
503 value are the same as for the parent routine. */
504
505 static bool
expand_subword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)506 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
507 rtx outof_input, rtx into_input, rtx op1,
508 rtx outof_target, rtx into_target,
509 int unsignedp, enum optab_methods methods,
510 unsigned HOST_WIDE_INT shift_mask)
511 {
512 optab reverse_unsigned_shift, unsigned_shift;
513 rtx tmp, carries;
514
515 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
516 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
517
518 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
519 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
520 the opposite direction to BINOPTAB. */
521 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
522 {
523 carries = outof_input;
524 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
525 op1_mode), op1_mode);
526 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
527 0, true, methods);
528 }
529 else
530 {
531 /* We must avoid shifting by BITS_PER_WORD bits since that is either
532 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
533 has unknown behavior. Do a single shift first, then shift by the
534 remainder. It's OK to use ~OP1 as the remainder if shift counts
535 are truncated to the mode size. */
536 carries = expand_binop (word_mode, reverse_unsigned_shift,
537 outof_input, const1_rtx, 0, unsignedp, methods);
538 if (shift_mask == BITS_PER_WORD - 1)
539 {
540 tmp = immed_wide_int_const
541 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
542 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
543 0, true, methods);
544 }
545 else
546 {
547 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
548 op1_mode), op1_mode);
549 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
550 0, true, methods);
551 }
552 }
553 if (tmp == 0 || carries == 0)
554 return false;
555 carries = expand_binop (word_mode, reverse_unsigned_shift,
556 carries, tmp, 0, unsignedp, methods);
557 if (carries == 0)
558 return false;
559
560 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
561 so the result can go directly into INTO_TARGET if convenient. */
562 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
563 into_target, unsignedp, methods);
564 if (tmp == 0)
565 return false;
566
567 /* Now OR in the bits carried over from OUTOF_INPUT. */
568 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
569 into_target, unsignedp, methods))
570 return false;
571
572 /* Use a standard word_mode shift for the out-of half. */
573 if (outof_target != 0)
574 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
575 outof_target, unsignedp, methods))
576 return false;
577
578 return true;
579 }
580
581
582 /* Try implementing expand_doubleword_shift using conditional moves.
583 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
584 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
585 are the shift counts to use in the former and latter case. All other
586 arguments are the same as the parent routine. */
587
588 static bool
expand_doubleword_shift_condmove(scalar_int_mode op1_mode,optab binoptab,enum rtx_code cmp_code,rtx cmp1,rtx cmp2,rtx outof_input,rtx into_input,rtx subword_op1,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)589 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
590 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
591 rtx outof_input, rtx into_input,
592 rtx subword_op1, rtx superword_op1,
593 rtx outof_target, rtx into_target,
594 int unsignedp, enum optab_methods methods,
595 unsigned HOST_WIDE_INT shift_mask)
596 {
597 rtx outof_superword, into_superword;
598
599 /* Put the superword version of the output into OUTOF_SUPERWORD and
600 INTO_SUPERWORD. */
601 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
602 if (outof_target != 0 && subword_op1 == superword_op1)
603 {
604 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
605 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
606 into_superword = outof_target;
607 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
608 outof_superword, 0, unsignedp, methods))
609 return false;
610 }
611 else
612 {
613 into_superword = gen_reg_rtx (word_mode);
614 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
615 outof_superword, into_superword,
616 unsignedp, methods))
617 return false;
618 }
619
620 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
621 if (!expand_subword_shift (op1_mode, binoptab,
622 outof_input, into_input, subword_op1,
623 outof_target, into_target,
624 unsignedp, methods, shift_mask))
625 return false;
626
627 /* Select between them. Do the INTO half first because INTO_SUPERWORD
628 might be the current value of OUTOF_TARGET. */
629 if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
630 into_target, into_superword, word_mode, false))
631 return false;
632
633 if (outof_target != 0)
634 if (!emit_conditional_move (outof_target,
635 { cmp_code, cmp1, cmp2, op1_mode },
636 outof_target, outof_superword,
637 word_mode, false))
638 return false;
639
640 return true;
641 }
642
643 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
644 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
645 input operand; the shift moves bits in the direction OUTOF_INPUT->
646 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
647 of the target. OP1 is the shift count and OP1_MODE is its mode.
648 If OP1 is constant, it will have been truncated as appropriate
649 and is known to be nonzero.
650
651 If SHIFT_MASK is zero, the result of word shifts is undefined when the
652 shift count is outside the range [0, BITS_PER_WORD). This routine must
653 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
654
655 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
656 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
657 fill with zeros or sign bits as appropriate.
658
659 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
660 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
661 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
662 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
663 are undefined.
664
665 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
666 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
667 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
668 function wants to calculate it itself.
669
670 Return true if the shift could be successfully synthesized. */
671
672 static bool
expand_doubleword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)673 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
674 rtx outof_input, rtx into_input, rtx op1,
675 rtx outof_target, rtx into_target,
676 int unsignedp, enum optab_methods methods,
677 unsigned HOST_WIDE_INT shift_mask)
678 {
679 rtx superword_op1, tmp, cmp1, cmp2;
680 enum rtx_code cmp_code;
681
682 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
683 fill the result with sign or zero bits as appropriate. If so, the value
684 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
685 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
686 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
687
688 This isn't worthwhile for constant shifts since the optimizers will
689 cope better with in-range shift counts. */
690 if (shift_mask >= BITS_PER_WORD
691 && outof_target != 0
692 && !CONSTANT_P (op1))
693 {
694 if (!expand_doubleword_shift (op1_mode, binoptab,
695 outof_input, into_input, op1,
696 0, into_target,
697 unsignedp, methods, shift_mask))
698 return false;
699 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
700 outof_target, unsignedp, methods))
701 return false;
702 return true;
703 }
704
705 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
706 is true when the effective shift value is less than BITS_PER_WORD.
707 Set SUPERWORD_OP1 to the shift count that should be used to shift
708 OUTOF_INPUT into INTO_TARGET when the condition is false. */
709 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
710 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
711 {
712 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
713 is a subword shift count. */
714 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
715 0, true, methods);
716 cmp2 = CONST0_RTX (op1_mode);
717 cmp_code = EQ;
718 superword_op1 = op1;
719 }
720 else
721 {
722 /* Set CMP1 to OP1 - BITS_PER_WORD. */
723 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
724 0, true, methods);
725 cmp2 = CONST0_RTX (op1_mode);
726 cmp_code = LT;
727 superword_op1 = cmp1;
728 }
729 if (cmp1 == 0)
730 return false;
731
732 /* If we can compute the condition at compile time, pick the
733 appropriate subroutine. */
734 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
735 if (tmp != 0 && CONST_INT_P (tmp))
736 {
737 if (tmp == const0_rtx)
738 return expand_superword_shift (binoptab, outof_input, superword_op1,
739 outof_target, into_target,
740 unsignedp, methods);
741 else
742 return expand_subword_shift (op1_mode, binoptab,
743 outof_input, into_input, op1,
744 outof_target, into_target,
745 unsignedp, methods, shift_mask);
746 }
747
748 /* Try using conditional moves to generate straight-line code. */
749 if (HAVE_conditional_move)
750 {
751 rtx_insn *start = get_last_insn ();
752 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
753 cmp_code, cmp1, cmp2,
754 outof_input, into_input,
755 op1, superword_op1,
756 outof_target, into_target,
757 unsignedp, methods, shift_mask))
758 return true;
759 delete_insns_since (start);
760 }
761
762 /* As a last resort, use branches to select the correct alternative. */
763 rtx_code_label *subword_label = gen_label_rtx ();
764 rtx_code_label *done_label = gen_label_rtx ();
765
766 NO_DEFER_POP;
767 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
768 0, 0, subword_label,
769 profile_probability::uninitialized ());
770 OK_DEFER_POP;
771
772 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
773 outof_target, into_target,
774 unsignedp, methods))
775 return false;
776
777 emit_jump_insn (targetm.gen_jump (done_label));
778 emit_barrier ();
779 emit_label (subword_label);
780
781 if (!expand_subword_shift (op1_mode, binoptab,
782 outof_input, into_input, op1,
783 outof_target, into_target,
784 unsignedp, methods, shift_mask))
785 return false;
786
787 emit_label (done_label);
788 return true;
789 }
790
791 /* Subroutine of expand_binop. Perform a double word multiplication of
792 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
793 as the target's word_mode. This function return NULL_RTX if anything
794 goes wrong, in which case it may have already emitted instructions
795 which need to be deleted.
796
797 If we want to multiply two two-word values and have normal and widening
798 multiplies of single-word values, we can do this with three smaller
799 multiplications.
800
801 The multiplication proceeds as follows:
802 _______________________
803 [__op0_high_|__op0_low__]
804 _______________________
805 * [__op1_high_|__op1_low__]
806 _______________________________________________
807 _______________________
808 (1) [__op0_low__*__op1_low__]
809 _______________________
810 (2a) [__op0_low__*__op1_high_]
811 _______________________
812 (2b) [__op0_high_*__op1_low__]
813 _______________________
814 (3) [__op0_high_*__op1_high_]
815
816
817 This gives a 4-word result. Since we are only interested in the
818 lower 2 words, partial result (3) and the upper words of (2a) and
819 (2b) don't need to be calculated. Hence (2a) and (2b) can be
820 calculated using non-widening multiplication.
821
822 (1), however, needs to be calculated with an unsigned widening
823 multiplication. If this operation is not directly supported we
824 try using a signed widening multiplication and adjust the result.
825 This adjustment works as follows:
826
827 If both operands are positive then no adjustment is needed.
828
829 If the operands have different signs, for example op0_low < 0 and
830 op1_low >= 0, the instruction treats the most significant bit of
831 op0_low as a sign bit instead of a bit with significance
832 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
833 with 2**BITS_PER_WORD - op0_low, and two's complements the
834 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
835 the result.
836
837 Similarly, if both operands are negative, we need to add
838 (op0_low + op1_low) * 2**BITS_PER_WORD.
839
840 We use a trick to adjust quickly. We logically shift op0_low right
841 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
842 op0_high (op1_high) before it is used to calculate 2b (2a). If no
843 logical shift exists, we do an arithmetic right shift and subtract
844 the 0 or -1. */
845
846 static rtx
expand_doubleword_mult(machine_mode mode,rtx op0,rtx op1,rtx target,bool umulp,enum optab_methods methods)847 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
848 bool umulp, enum optab_methods methods)
849 {
850 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
851 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
852 rtx wordm1 = (umulp ? NULL_RTX
853 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
854 rtx product, adjust, product_high, temp;
855
856 rtx op0_high = operand_subword_force (op0, high, mode);
857 rtx op0_low = operand_subword_force (op0, low, mode);
858 rtx op1_high = operand_subword_force (op1, high, mode);
859 rtx op1_low = operand_subword_force (op1, low, mode);
860
861 /* If we're using an unsigned multiply to directly compute the product
862 of the low-order words of the operands and perform any required
863 adjustments of the operands, we begin by trying two more multiplications
864 and then computing the appropriate sum.
865
866 We have checked above that the required addition is provided.
867 Full-word addition will normally always succeed, especially if
868 it is provided at all, so we don't worry about its failure. The
869 multiplication may well fail, however, so we do handle that. */
870
871 if (!umulp)
872 {
873 /* ??? This could be done with emit_store_flag where available. */
874 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
875 NULL_RTX, 1, methods);
876 if (temp)
877 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
878 NULL_RTX, 0, OPTAB_DIRECT);
879 else
880 {
881 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
882 NULL_RTX, 0, methods);
883 if (!temp)
884 return NULL_RTX;
885 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
886 NULL_RTX, 0, OPTAB_DIRECT);
887 }
888
889 if (!op0_high)
890 return NULL_RTX;
891 }
892
893 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
894 NULL_RTX, 0, OPTAB_DIRECT);
895 if (!adjust)
896 return NULL_RTX;
897
898 /* OP0_HIGH should now be dead. */
899
900 if (!umulp)
901 {
902 /* ??? This could be done with emit_store_flag where available. */
903 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
904 NULL_RTX, 1, methods);
905 if (temp)
906 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
907 NULL_RTX, 0, OPTAB_DIRECT);
908 else
909 {
910 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
911 NULL_RTX, 0, methods);
912 if (!temp)
913 return NULL_RTX;
914 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
915 NULL_RTX, 0, OPTAB_DIRECT);
916 }
917
918 if (!op1_high)
919 return NULL_RTX;
920 }
921
922 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
923 NULL_RTX, 0, OPTAB_DIRECT);
924 if (!temp)
925 return NULL_RTX;
926
927 /* OP1_HIGH should now be dead. */
928
929 adjust = expand_binop (word_mode, add_optab, adjust, temp,
930 NULL_RTX, 0, OPTAB_DIRECT);
931
932 if (target && !REG_P (target))
933 target = NULL_RTX;
934
935 /* *_widen_optab needs to determine operand mode, make sure at least
936 one operand has non-VOID mode. */
937 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
938 op0_low = force_reg (word_mode, op0_low);
939
940 if (umulp)
941 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
942 target, 1, OPTAB_DIRECT);
943 else
944 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
945 target, 1, OPTAB_DIRECT);
946
947 if (!product)
948 return NULL_RTX;
949
950 product_high = operand_subword (product, high, 1, mode);
951 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
952 NULL_RTX, 0, OPTAB_DIRECT);
953 emit_move_insn (product_high, adjust);
954 return product;
955 }
956
957 /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
958 constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
959 (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
960 computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
961 + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
962 depends on the bit value, if 2, then carry from the addition needs to be
963 added too, i.e. like:
964 sum += __builtin_add_overflow (low, high, &sum)
965
966 Optimize signed double-word OP0 % OP1 similarly, just apply some correction
967 factor to the sum before doing unsigned remainder, in the form of
968 sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
969 then perform unsigned
970 remainder = sum % OP1;
971 and finally
972 remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
973
974 static rtx
expand_doubleword_mod(machine_mode mode,rtx op0,rtx op1,bool unsignedp)975 expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
976 {
977 if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
978 return NULL_RTX;
979
980 rtx_insn *last = get_last_insn ();
981 for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
982 {
983 wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
984 if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
985 continue;
986 rtx sum = NULL_RTX, mask = NULL_RTX;
987 if (bit == BITS_PER_WORD)
988 {
989 /* For signed modulo we need to add correction to the sum
990 and that might again overflow. */
991 if (!unsignedp)
992 continue;
993 if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
994 continue;
995 tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
996 if (wtype == NULL_TREE)
997 continue;
998 tree ctype = build_complex_type (wtype);
999 if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1000 continue;
1001 machine_mode cmode = TYPE_MODE (ctype);
1002 rtx op00 = operand_subword_force (op0, 0, mode);
1003 rtx op01 = operand_subword_force (op0, 1, mode);
1004 rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1005 gen_reg_rtx (word_mode));
1006 tree lhs = make_tree (ctype, cres);
1007 tree arg0 = make_tree (wtype, op00);
1008 tree arg1 = make_tree (wtype, op01);
1009 expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1010 arg1, true, true, true, false, NULL);
1011 sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1012 XEXP (cres, 1), NULL_RTX, 1,
1013 OPTAB_DIRECT);
1014 if (sum == NULL_RTX)
1015 return NULL_RTX;
1016 }
1017 else
1018 {
1019 /* Code below uses GEN_INT, so we need the masks to be representable
1020 in HOST_WIDE_INTs. */
1021 if (bit >= HOST_BITS_PER_WIDE_INT)
1022 continue;
1023 /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1024 overflow. Consider 64-bit -1ULL for word size 32, if we add
1025 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1026 if (bit == BITS_PER_WORD - 1)
1027 continue;
1028
1029 int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1030 rtx sum_corr = NULL_RTX;
1031
1032 if (!unsignedp)
1033 {
1034 /* For signed modulo, compute it as unsigned modulo of
1035 sum with a correction added to it if OP0 is negative,
1036 such that the result can be computed as unsigned
1037 remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1038 w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1039 wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
1040 wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
1041 /* wmod2 == -wmod1. */
1042 wmod2 = wmod2 + (INTVAL (op1) - 1);
1043 if (wi::ne_p (wmod1, wmod2))
1044 {
1045 wide_int wcorr = wmod2 - wmod1;
1046 if (wi::neg_p (w))
1047 wcorr = wcorr + INTVAL (op1);
1048 /* Now verify if the count sums can't overflow, and punt
1049 if they could. */
1050 w = wi::mask (bit, false, 2 * BITS_PER_WORD);
1051 w = w * (count - 1);
1052 w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
1053 false, 2 * BITS_PER_WORD);
1054 w = w + wcorr;
1055 w = wi::lrshift (w, BITS_PER_WORD);
1056 if (wi::ne_p (w, 0))
1057 continue;
1058
1059 mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1060 mode);
1061 mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1062 GEN_INT (BITS_PER_WORD - 1),
1063 NULL_RTX, 0, OPTAB_DIRECT);
1064 if (mask == NULL_RTX)
1065 return NULL_RTX;
1066 sum_corr = immed_wide_int_const (wcorr, word_mode);
1067 sum_corr = expand_simple_binop (word_mode, AND, mask,
1068 sum_corr, NULL_RTX, 1,
1069 OPTAB_DIRECT);
1070 if (sum_corr == NULL_RTX)
1071 return NULL_RTX;
1072 }
1073 }
1074
1075 for (int i = 0; i < count; i++)
1076 {
1077 rtx v = op0;
1078 if (i)
1079 v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1080 NULL_RTX, 1, OPTAB_DIRECT);
1081 if (v == NULL_RTX)
1082 return NULL_RTX;
1083 v = lowpart_subreg (word_mode, v, mode);
1084 if (v == NULL_RTX)
1085 return NULL_RTX;
1086 if (i != count - 1)
1087 v = expand_simple_binop (word_mode, AND, v,
1088 GEN_INT ((HOST_WIDE_INT_1U << bit)
1089 - 1), NULL_RTX, 1,
1090 OPTAB_DIRECT);
1091 if (v == NULL_RTX)
1092 return NULL_RTX;
1093 if (sum == NULL_RTX)
1094 sum = v;
1095 else
1096 sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1097 1, OPTAB_DIRECT);
1098 if (sum == NULL_RTX)
1099 return NULL_RTX;
1100 }
1101 if (sum_corr)
1102 {
1103 sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1104 NULL_RTX, 1, OPTAB_DIRECT);
1105 if (sum == NULL_RTX)
1106 return NULL_RTX;
1107 }
1108 }
1109 rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1110 gen_int_mode (INTVAL (op1), word_mode),
1111 NULL_RTX, 1, OPTAB_DIRECT);
1112 if (remainder == NULL_RTX)
1113 return NULL_RTX;
1114
1115 if (!unsignedp)
1116 {
1117 if (mask == NULL_RTX)
1118 {
1119 mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1120 mode);
1121 mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1122 GEN_INT (BITS_PER_WORD - 1),
1123 NULL_RTX, 0, OPTAB_DIRECT);
1124 if (mask == NULL_RTX)
1125 return NULL_RTX;
1126 }
1127 mask = expand_simple_binop (word_mode, AND, mask,
1128 gen_int_mode (1 - INTVAL (op1),
1129 word_mode),
1130 NULL_RTX, 1, OPTAB_DIRECT);
1131 if (mask == NULL_RTX)
1132 return NULL_RTX;
1133 remainder = expand_simple_binop (word_mode, PLUS, remainder,
1134 mask, NULL_RTX, 1, OPTAB_DIRECT);
1135 if (remainder == NULL_RTX)
1136 return NULL_RTX;
1137 }
1138
1139 remainder = convert_modes (mode, word_mode, remainder, unsignedp);
1140 /* Punt if we need any library calls. */
1141 if (last)
1142 last = NEXT_INSN (last);
1143 else
1144 last = get_insns ();
1145 for (; last; last = NEXT_INSN (last))
1146 if (CALL_P (last))
1147 return NULL_RTX;
1148 return remainder;
1149 }
1150 return NULL_RTX;
1151 }
1152
1153 /* Similarly to the above function, but compute both quotient and remainder.
1154 Quotient can be computed from the remainder as:
1155 rem = op0 % op1; // Handled using expand_doubleword_mod
1156 quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1157 // 2 * BITS_PER_WORD
1158
1159 We can also handle cases where op1 is a multiple of power of two constant
1160 and constant handled by expand_doubleword_mod.
1161 op11 = 1 << __builtin_ctz (op1);
1162 op12 = op1 / op11;
1163 rem1 = op0 % op12; // Handled using expand_doubleword_mod
1164 quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1165 // 2 * BITS_PER_WORD
1166 rem = (quot1 % op11) * op12 + rem1;
1167 quot = quot1 / op11; */
1168
1169 rtx
expand_doubleword_divmod(machine_mode mode,rtx op0,rtx op1,rtx * rem,bool unsignedp)1170 expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1171 bool unsignedp)
1172 {
1173 *rem = NULL_RTX;
1174
1175 /* Negative dividend should have been optimized into positive,
1176 similarly modulo by 1 and modulo by power of two is optimized
1177 differently too. */
1178 if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1179 return NULL_RTX;
1180
1181 rtx op11 = const1_rtx;
1182 rtx op12 = op1;
1183 if ((INTVAL (op1) & 1) == 0)
1184 {
1185 int bit = ctz_hwi (INTVAL (op1));
1186 op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1187 op12 = GEN_INT (INTVAL (op1) >> bit);
1188 }
1189
1190 rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
1191 if (rem1 == NULL_RTX)
1192 return NULL_RTX;
1193
1194 int prec = 2 * BITS_PER_WORD;
1195 wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
1196 wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
1197 wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
1198 rtx inv = immed_wide_int_const (m, mode);
1199
1200 rtx_insn *last = get_last_insn ();
1201 rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1202 NULL_RTX, unsignedp, OPTAB_DIRECT);
1203 if (quot1 == NULL_RTX)
1204 return NULL_RTX;
1205
1206 quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1207 NULL_RTX, unsignedp, OPTAB_DIRECT);
1208 if (quot1 == NULL_RTX)
1209 return NULL_RTX;
1210
1211 if (op11 != const1_rtx)
1212 {
1213 rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1214 NULL_RTX, unsignedp, OPTAB_DIRECT);
1215 if (rem2 == NULL_RTX)
1216 return NULL_RTX;
1217
1218 rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1219 unsignedp, OPTAB_DIRECT);
1220 if (rem2 == NULL_RTX)
1221 return NULL_RTX;
1222
1223 rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1224 unsignedp, OPTAB_DIRECT);
1225 if (rem2 == NULL_RTX)
1226 return NULL_RTX;
1227
1228 rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1229 NULL_RTX, unsignedp, OPTAB_DIRECT);
1230 if (quot2 == NULL_RTX)
1231 return NULL_RTX;
1232
1233 rem1 = rem2;
1234 quot1 = quot2;
1235 }
1236
1237 /* Punt if we need any library calls. */
1238 if (last)
1239 last = NEXT_INSN (last);
1240 else
1241 last = get_insns ();
1242 for (; last; last = NEXT_INSN (last))
1243 if (CALL_P (last))
1244 return NULL_RTX;
1245
1246 *rem = rem1;
1247 return quot1;
1248 }
1249
1250 /* Wrapper around expand_binop which takes an rtx code to specify
1251 the operation to perform, not an optab pointer. All other
1252 arguments are the same. */
1253 rtx
expand_simple_binop(machine_mode mode,enum rtx_code code,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1254 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1255 rtx op1, rtx target, int unsignedp,
1256 enum optab_methods methods)
1257 {
1258 optab binop = code_to_optab (code);
1259 gcc_assert (binop);
1260
1261 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1262 }
1263
1264 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1265 binop. Order them according to commutative_operand_precedence and, if
1266 possible, try to put TARGET or a pseudo first. */
1267 static bool
swap_commutative_operands_with_target(rtx target,rtx op0,rtx op1)1268 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1269 {
1270 int op0_prec = commutative_operand_precedence (op0);
1271 int op1_prec = commutative_operand_precedence (op1);
1272
1273 if (op0_prec < op1_prec)
1274 return true;
1275
1276 if (op0_prec > op1_prec)
1277 return false;
1278
1279 /* With equal precedence, both orders are ok, but it is better if the
1280 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1281 if (target == 0 || REG_P (target))
1282 return (REG_P (op1) && !REG_P (op0)) || target == op1;
1283 else
1284 return rtx_equal_p (op1, target);
1285 }
1286
1287 /* Return true if BINOPTAB implements a shift operation. */
1288
1289 static bool
shift_optab_p(optab binoptab)1290 shift_optab_p (optab binoptab)
1291 {
1292 switch (optab_to_code (binoptab))
1293 {
1294 case ASHIFT:
1295 case SS_ASHIFT:
1296 case US_ASHIFT:
1297 case ASHIFTRT:
1298 case LSHIFTRT:
1299 case ROTATE:
1300 case ROTATERT:
1301 return true;
1302
1303 default:
1304 return false;
1305 }
1306 }
1307
1308 /* Return true if BINOPTAB implements a commutative binary operation. */
1309
1310 static bool
commutative_optab_p(optab binoptab)1311 commutative_optab_p (optab binoptab)
1312 {
1313 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1314 || binoptab == smul_widen_optab
1315 || binoptab == umul_widen_optab
1316 || binoptab == smul_highpart_optab
1317 || binoptab == umul_highpart_optab);
1318 }
1319
1320 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1321 optimizing, and if the operand is a constant that costs more than
1322 1 instruction, force the constant into a register and return that
1323 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1324
1325 static rtx
avoid_expensive_constant(machine_mode mode,optab binoptab,int opn,rtx x,bool unsignedp)1326 avoid_expensive_constant (machine_mode mode, optab binoptab,
1327 int opn, rtx x, bool unsignedp)
1328 {
1329 bool speed = optimize_insn_for_speed_p ();
1330
1331 if (mode != VOIDmode
1332 && optimize
1333 && CONSTANT_P (x)
1334 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1335 > set_src_cost (x, mode, speed)))
1336 {
1337 if (CONST_INT_P (x))
1338 {
1339 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1340 if (intval != INTVAL (x))
1341 x = GEN_INT (intval);
1342 }
1343 else
1344 x = convert_modes (mode, VOIDmode, x, unsignedp);
1345 x = force_reg (mode, x);
1346 }
1347 return x;
1348 }
1349
1350 /* Helper function for expand_binop: handle the case where there
1351 is an insn ICODE that directly implements the indicated operation.
1352 Returns null if this is not possible. */
1353 static rtx
expand_binop_directly(enum insn_code icode,machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods,rtx_insn * last)1354 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1355 rtx op0, rtx op1,
1356 rtx target, int unsignedp, enum optab_methods methods,
1357 rtx_insn *last)
1358 {
1359 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1360 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1361 machine_mode mode0, mode1, tmp_mode;
1362 class expand_operand ops[3];
1363 bool commutative_p;
1364 rtx_insn *pat;
1365 rtx xop0 = op0, xop1 = op1;
1366 bool canonicalize_op1 = false;
1367
1368 /* If it is a commutative operator and the modes would match
1369 if we would swap the operands, we can save the conversions. */
1370 commutative_p = commutative_optab_p (binoptab);
1371 if (commutative_p
1372 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1373 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1374 std::swap (xop0, xop1);
1375
1376 /* If we are optimizing, force expensive constants into a register. */
1377 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1378 if (!shift_optab_p (binoptab))
1379 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1380 else
1381 /* Shifts and rotates often use a different mode for op1 from op0;
1382 for VOIDmode constants we don't know the mode, so force it
1383 to be canonicalized using convert_modes. */
1384 canonicalize_op1 = true;
1385
1386 /* In case the insn wants input operands in modes different from
1387 those of the actual operands, convert the operands. It would
1388 seem that we don't need to convert CONST_INTs, but we do, so
1389 that they're properly zero-extended, sign-extended or truncated
1390 for their mode. */
1391
1392 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1393 if (xmode0 != VOIDmode && xmode0 != mode0)
1394 {
1395 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1396 mode0 = xmode0;
1397 }
1398
1399 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1400 ? GET_MODE (xop1) : mode);
1401 if (xmode1 != VOIDmode && xmode1 != mode1)
1402 {
1403 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1404 mode1 = xmode1;
1405 }
1406
1407 /* If operation is commutative,
1408 try to make the first operand a register.
1409 Even better, try to make it the same as the target.
1410 Also try to make the last operand a constant. */
1411 if (commutative_p
1412 && swap_commutative_operands_with_target (target, xop0, xop1))
1413 std::swap (xop0, xop1);
1414
1415 /* Now, if insn's predicates don't allow our operands, put them into
1416 pseudo regs. */
1417
1418 if (binoptab == vec_pack_trunc_optab
1419 || binoptab == vec_pack_usat_optab
1420 || binoptab == vec_pack_ssat_optab
1421 || binoptab == vec_pack_ufix_trunc_optab
1422 || binoptab == vec_pack_sfix_trunc_optab
1423 || binoptab == vec_packu_float_optab
1424 || binoptab == vec_packs_float_optab)
1425 {
1426 /* The mode of the result is different then the mode of the
1427 arguments. */
1428 tmp_mode = insn_data[(int) icode].operand[0].mode;
1429 if (VECTOR_MODE_P (mode)
1430 && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1431 {
1432 delete_insns_since (last);
1433 return NULL_RTX;
1434 }
1435 }
1436 else
1437 tmp_mode = mode;
1438
1439 create_output_operand (&ops[0], target, tmp_mode);
1440 create_input_operand (&ops[1], xop0, mode0);
1441 create_input_operand (&ops[2], xop1, mode1);
1442 pat = maybe_gen_insn (icode, 3, ops);
1443 if (pat)
1444 {
1445 /* If PAT is composed of more than one insn, try to add an appropriate
1446 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1447 operand, call expand_binop again, this time without a target. */
1448 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1449 && ! add_equal_note (pat, ops[0].value,
1450 optab_to_code (binoptab),
1451 ops[1].value, ops[2].value, mode0))
1452 {
1453 delete_insns_since (last);
1454 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1455 unsignedp, methods);
1456 }
1457
1458 emit_insn (pat);
1459 return ops[0].value;
1460 }
1461 delete_insns_since (last);
1462 return NULL_RTX;
1463 }
1464
1465 /* Generate code to perform an operation specified by BINOPTAB
1466 on operands OP0 and OP1, with result having machine-mode MODE.
1467
1468 UNSIGNEDP is for the case where we have to widen the operands
1469 to perform the operation. It says to use zero-extension.
1470
1471 If TARGET is nonzero, the value
1472 is generated there, if it is convenient to do so.
1473 In all cases an rtx is returned for the locus of the value;
1474 this may or may not be TARGET. */
1475
1476 rtx
expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1477 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1478 rtx target, int unsignedp, enum optab_methods methods)
1479 {
1480 enum optab_methods next_methods
1481 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1482 ? OPTAB_WIDEN : methods);
1483 enum mode_class mclass;
1484 enum insn_code icode;
1485 machine_mode wider_mode;
1486 scalar_int_mode int_mode;
1487 rtx libfunc;
1488 rtx temp;
1489 rtx_insn *entry_last = get_last_insn ();
1490 rtx_insn *last;
1491
1492 mclass = GET_MODE_CLASS (mode);
1493
1494 /* If subtracting an integer constant, convert this into an addition of
1495 the negated constant. */
1496
1497 if (binoptab == sub_optab && CONST_INT_P (op1))
1498 {
1499 op1 = negate_rtx (mode, op1);
1500 binoptab = add_optab;
1501 }
1502 /* For shifts, constant invalid op1 might be expanded from different
1503 mode than MODE. As those are invalid, force them to a register
1504 to avoid further problems during expansion. */
1505 else if (CONST_INT_P (op1)
1506 && shift_optab_p (binoptab)
1507 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1508 {
1509 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1510 op1 = force_reg (GET_MODE_INNER (mode), op1);
1511 }
1512
1513 /* Record where to delete back to if we backtrack. */
1514 last = get_last_insn ();
1515
1516 /* If we can do it with a three-operand insn, do so. */
1517
1518 if (methods != OPTAB_MUST_WIDEN)
1519 {
1520 if (convert_optab_p (binoptab))
1521 {
1522 machine_mode from_mode = widened_mode (mode, op0, op1);
1523 icode = find_widening_optab_handler (binoptab, mode, from_mode);
1524 }
1525 else
1526 icode = optab_handler (binoptab, mode);
1527 if (icode != CODE_FOR_nothing)
1528 {
1529 temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1530 target, unsignedp, methods, last);
1531 if (temp)
1532 return temp;
1533 }
1534 }
1535
1536 /* If we were trying to rotate, and that didn't work, try rotating
1537 the other direction before falling back to shifts and bitwise-or. */
1538 if (((binoptab == rotl_optab
1539 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1540 || (binoptab == rotr_optab
1541 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1542 && is_int_mode (mode, &int_mode))
1543 {
1544 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1545 rtx newop1;
1546 unsigned int bits = GET_MODE_PRECISION (int_mode);
1547
1548 if (CONST_INT_P (op1))
1549 newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1550 else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1551 newop1 = negate_rtx (GET_MODE (op1), op1);
1552 else
1553 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1554 gen_int_mode (bits, GET_MODE (op1)), op1,
1555 NULL_RTX, unsignedp, OPTAB_DIRECT);
1556
1557 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1558 target, unsignedp, methods, last);
1559 if (temp)
1560 return temp;
1561 }
1562
1563 /* If this is a multiply, see if we can do a widening operation that
1564 takes operands of this mode and makes a wider mode. */
1565
1566 if (binoptab == smul_optab
1567 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1568 && (convert_optab_handler ((unsignedp
1569 ? umul_widen_optab
1570 : smul_widen_optab),
1571 wider_mode, mode) != CODE_FOR_nothing))
1572 {
1573 /* *_widen_optab needs to determine operand mode, make sure at least
1574 one operand has non-VOID mode. */
1575 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1576 op0 = force_reg (mode, op0);
1577 temp = expand_binop (wider_mode,
1578 unsignedp ? umul_widen_optab : smul_widen_optab,
1579 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1580
1581 if (temp != 0)
1582 {
1583 if (GET_MODE_CLASS (mode) == MODE_INT
1584 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1585 return gen_lowpart (mode, temp);
1586 else
1587 return convert_to_mode (mode, temp, unsignedp);
1588 }
1589 }
1590
1591 /* If this is a vector shift by a scalar, see if we can do a vector
1592 shift by a vector. If so, broadcast the scalar into a vector. */
1593 if (mclass == MODE_VECTOR_INT)
1594 {
1595 optab otheroptab = unknown_optab;
1596
1597 if (binoptab == ashl_optab)
1598 otheroptab = vashl_optab;
1599 else if (binoptab == ashr_optab)
1600 otheroptab = vashr_optab;
1601 else if (binoptab == lshr_optab)
1602 otheroptab = vlshr_optab;
1603 else if (binoptab == rotl_optab)
1604 otheroptab = vrotl_optab;
1605 else if (binoptab == rotr_optab)
1606 otheroptab = vrotr_optab;
1607
1608 if (otheroptab
1609 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1610 {
1611 /* The scalar may have been extended to be too wide. Truncate
1612 it back to the proper size to fit in the broadcast vector. */
1613 scalar_mode inner_mode = GET_MODE_INNER (mode);
1614 if (!CONST_INT_P (op1)
1615 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1616 > GET_MODE_BITSIZE (inner_mode)))
1617 op1 = force_reg (inner_mode,
1618 simplify_gen_unary (TRUNCATE, inner_mode, op1,
1619 GET_MODE (op1)));
1620 rtx vop1 = expand_vector_broadcast (mode, op1);
1621 if (vop1)
1622 {
1623 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1624 target, unsignedp, methods, last);
1625 if (temp)
1626 return temp;
1627 }
1628 }
1629 }
1630
1631 /* Look for a wider mode of the same class for which we think we
1632 can open-code the operation. Check for a widening multiply at the
1633 wider mode as well. */
1634
1635 if (CLASS_HAS_WIDER_MODES_P (mclass)
1636 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1637 FOR_EACH_WIDER_MODE (wider_mode, mode)
1638 {
1639 machine_mode next_mode;
1640 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1641 || (binoptab == smul_optab
1642 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1643 && (find_widening_optab_handler ((unsignedp
1644 ? umul_widen_optab
1645 : smul_widen_optab),
1646 next_mode, mode)
1647 != CODE_FOR_nothing)))
1648 {
1649 rtx xop0 = op0, xop1 = op1;
1650 int no_extend = 0;
1651
1652 /* For certain integer operations, we need not actually extend
1653 the narrow operands, as long as we will truncate
1654 the results to the same narrowness. */
1655
1656 if ((binoptab == ior_optab || binoptab == and_optab
1657 || binoptab == xor_optab
1658 || binoptab == add_optab || binoptab == sub_optab
1659 || binoptab == smul_optab || binoptab == ashl_optab)
1660 && mclass == MODE_INT)
1661 {
1662 no_extend = 1;
1663 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1664 xop0, unsignedp);
1665 if (binoptab != ashl_optab)
1666 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1667 xop1, unsignedp);
1668 }
1669
1670 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1671
1672 /* The second operand of a shift must always be extended. */
1673 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1674 no_extend && binoptab != ashl_optab);
1675
1676 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1677 unsignedp, OPTAB_DIRECT);
1678 if (temp)
1679 {
1680 if (mclass != MODE_INT
1681 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1682 {
1683 if (target == 0)
1684 target = gen_reg_rtx (mode);
1685 convert_move (target, temp, 0);
1686 return target;
1687 }
1688 else
1689 return gen_lowpart (mode, temp);
1690 }
1691 else
1692 delete_insns_since (last);
1693 }
1694 }
1695
1696 /* If operation is commutative,
1697 try to make the first operand a register.
1698 Even better, try to make it the same as the target.
1699 Also try to make the last operand a constant. */
1700 if (commutative_optab_p (binoptab)
1701 && swap_commutative_operands_with_target (target, op0, op1))
1702 std::swap (op0, op1);
1703
1704 /* These can be done a word at a time. */
1705 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1706 && is_int_mode (mode, &int_mode)
1707 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1708 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1709 {
1710 int i;
1711 rtx_insn *insns;
1712
1713 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1714 won't be accurate, so use a new target. */
1715 if (target == 0
1716 || target == op0
1717 || target == op1
1718 || reg_overlap_mentioned_p (target, op0)
1719 || reg_overlap_mentioned_p (target, op1)
1720 || !valid_multiword_target_p (target))
1721 target = gen_reg_rtx (int_mode);
1722
1723 start_sequence ();
1724
1725 /* Do the actual arithmetic. */
1726 machine_mode op0_mode = GET_MODE (op0);
1727 machine_mode op1_mode = GET_MODE (op1);
1728 if (op0_mode == VOIDmode)
1729 op0_mode = int_mode;
1730 if (op1_mode == VOIDmode)
1731 op1_mode = int_mode;
1732 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1733 {
1734 rtx target_piece = operand_subword (target, i, 1, int_mode);
1735 rtx x = expand_binop (word_mode, binoptab,
1736 operand_subword_force (op0, i, op0_mode),
1737 operand_subword_force (op1, i, op1_mode),
1738 target_piece, unsignedp, next_methods);
1739
1740 if (x == 0)
1741 break;
1742
1743 if (target_piece != x)
1744 emit_move_insn (target_piece, x);
1745 }
1746
1747 insns = get_insns ();
1748 end_sequence ();
1749
1750 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1751 {
1752 emit_insn (insns);
1753 return target;
1754 }
1755 }
1756
1757 /* Synthesize double word shifts from single word shifts. */
1758 if ((binoptab == lshr_optab || binoptab == ashl_optab
1759 || binoptab == ashr_optab)
1760 && is_int_mode (mode, &int_mode)
1761 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1762 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1763 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1764 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1765 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1766 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1767 {
1768 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1769 scalar_int_mode op1_mode;
1770
1771 double_shift_mask = targetm.shift_truncation_mask (int_mode);
1772 shift_mask = targetm.shift_truncation_mask (word_mode);
1773 op1_mode = (GET_MODE (op1) != VOIDmode
1774 ? as_a <scalar_int_mode> (GET_MODE (op1))
1775 : word_mode);
1776
1777 /* Apply the truncation to constant shifts. */
1778 if (double_shift_mask > 0 && CONST_INT_P (op1))
1779 op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1780
1781 if (op1 == CONST0_RTX (op1_mode))
1782 return op0;
1783
1784 /* Make sure that this is a combination that expand_doubleword_shift
1785 can handle. See the comments there for details. */
1786 if (double_shift_mask == 0
1787 || (shift_mask == BITS_PER_WORD - 1
1788 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1789 {
1790 rtx_insn *insns;
1791 rtx into_target, outof_target;
1792 rtx into_input, outof_input;
1793 int left_shift, outof_word;
1794
1795 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1796 won't be accurate, so use a new target. */
1797 if (target == 0
1798 || target == op0
1799 || target == op1
1800 || reg_overlap_mentioned_p (target, op0)
1801 || reg_overlap_mentioned_p (target, op1)
1802 || !valid_multiword_target_p (target))
1803 target = gen_reg_rtx (int_mode);
1804
1805 start_sequence ();
1806
1807 /* OUTOF_* is the word we are shifting bits away from, and
1808 INTO_* is the word that we are shifting bits towards, thus
1809 they differ depending on the direction of the shift and
1810 WORDS_BIG_ENDIAN. */
1811
1812 left_shift = binoptab == ashl_optab;
1813 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1814
1815 outof_target = operand_subword (target, outof_word, 1, int_mode);
1816 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1817
1818 outof_input = operand_subword_force (op0, outof_word, int_mode);
1819 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1820
1821 if (expand_doubleword_shift (op1_mode, binoptab,
1822 outof_input, into_input, op1,
1823 outof_target, into_target,
1824 unsignedp, next_methods, shift_mask))
1825 {
1826 insns = get_insns ();
1827 end_sequence ();
1828
1829 emit_insn (insns);
1830 return target;
1831 }
1832 end_sequence ();
1833 }
1834 }
1835
1836 /* Synthesize double word rotates from single word shifts. */
1837 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1838 && is_int_mode (mode, &int_mode)
1839 && CONST_INT_P (op1)
1840 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1841 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1842 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1843 {
1844 rtx_insn *insns;
1845 rtx into_target, outof_target;
1846 rtx into_input, outof_input;
1847 rtx inter;
1848 int shift_count, left_shift, outof_word;
1849
1850 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1851 won't be accurate, so use a new target. Do this also if target is not
1852 a REG, first because having a register instead may open optimization
1853 opportunities, and second because if target and op0 happen to be MEMs
1854 designating the same location, we would risk clobbering it too early
1855 in the code sequence we generate below. */
1856 if (target == 0
1857 || target == op0
1858 || target == op1
1859 || !REG_P (target)
1860 || reg_overlap_mentioned_p (target, op0)
1861 || reg_overlap_mentioned_p (target, op1)
1862 || !valid_multiword_target_p (target))
1863 target = gen_reg_rtx (int_mode);
1864
1865 start_sequence ();
1866
1867 shift_count = INTVAL (op1);
1868
1869 /* OUTOF_* is the word we are shifting bits away from, and
1870 INTO_* is the word that we are shifting bits towards, thus
1871 they differ depending on the direction of the shift and
1872 WORDS_BIG_ENDIAN. */
1873
1874 left_shift = (binoptab == rotl_optab);
1875 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1876
1877 outof_target = operand_subword (target, outof_word, 1, int_mode);
1878 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1879
1880 outof_input = operand_subword_force (op0, outof_word, int_mode);
1881 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1882
1883 if (shift_count == BITS_PER_WORD)
1884 {
1885 /* This is just a word swap. */
1886 emit_move_insn (outof_target, into_input);
1887 emit_move_insn (into_target, outof_input);
1888 inter = const0_rtx;
1889 }
1890 else
1891 {
1892 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1893 HOST_WIDE_INT first_shift_count, second_shift_count;
1894 optab reverse_unsigned_shift, unsigned_shift;
1895
1896 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1897 ? lshr_optab : ashl_optab);
1898
1899 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1900 ? ashl_optab : lshr_optab);
1901
1902 if (shift_count > BITS_PER_WORD)
1903 {
1904 first_shift_count = shift_count - BITS_PER_WORD;
1905 second_shift_count = 2 * BITS_PER_WORD - shift_count;
1906 }
1907 else
1908 {
1909 first_shift_count = BITS_PER_WORD - shift_count;
1910 second_shift_count = shift_count;
1911 }
1912 rtx first_shift_count_rtx
1913 = gen_int_shift_amount (word_mode, first_shift_count);
1914 rtx second_shift_count_rtx
1915 = gen_int_shift_amount (word_mode, second_shift_count);
1916
1917 into_temp1 = expand_binop (word_mode, unsigned_shift,
1918 outof_input, first_shift_count_rtx,
1919 NULL_RTX, unsignedp, next_methods);
1920 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1921 into_input, second_shift_count_rtx,
1922 NULL_RTX, unsignedp, next_methods);
1923
1924 if (into_temp1 != 0 && into_temp2 != 0)
1925 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1926 into_target, unsignedp, next_methods);
1927 else
1928 inter = 0;
1929
1930 if (inter != 0 && inter != into_target)
1931 emit_move_insn (into_target, inter);
1932
1933 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1934 into_input, first_shift_count_rtx,
1935 NULL_RTX, unsignedp, next_methods);
1936 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1937 outof_input, second_shift_count_rtx,
1938 NULL_RTX, unsignedp, next_methods);
1939
1940 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1941 inter = expand_binop (word_mode, ior_optab,
1942 outof_temp1, outof_temp2,
1943 outof_target, unsignedp, next_methods);
1944
1945 if (inter != 0 && inter != outof_target)
1946 emit_move_insn (outof_target, inter);
1947 }
1948
1949 insns = get_insns ();
1950 end_sequence ();
1951
1952 if (inter != 0)
1953 {
1954 emit_insn (insns);
1955 return target;
1956 }
1957 }
1958
1959 /* These can be done a word at a time by propagating carries. */
1960 if ((binoptab == add_optab || binoptab == sub_optab)
1961 && is_int_mode (mode, &int_mode)
1962 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1963 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1964 {
1965 unsigned int i;
1966 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1967 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1968 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1969 rtx xop0, xop1, xtarget;
1970
1971 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1972 value is one of those, use it. Otherwise, use 1 since it is the
1973 one easiest to get. */
1974 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1975 int normalizep = STORE_FLAG_VALUE;
1976 #else
1977 int normalizep = 1;
1978 #endif
1979
1980 /* Prepare the operands. */
1981 xop0 = force_reg (int_mode, op0);
1982 xop1 = force_reg (int_mode, op1);
1983
1984 xtarget = gen_reg_rtx (int_mode);
1985
1986 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1987 target = xtarget;
1988
1989 /* Indicate for flow that the entire target reg is being set. */
1990 if (REG_P (target))
1991 emit_clobber (xtarget);
1992
1993 /* Do the actual arithmetic. */
1994 for (i = 0; i < nwords; i++)
1995 {
1996 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1997 rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1998 rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1999 rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2000 rtx x;
2001
2002 /* Main add/subtract of the input operands. */
2003 x = expand_binop (word_mode, binoptab,
2004 op0_piece, op1_piece,
2005 target_piece, unsignedp, next_methods);
2006 if (x == 0)
2007 break;
2008
2009 if (i + 1 < nwords)
2010 {
2011 /* Store carry from main add/subtract. */
2012 carry_out = gen_reg_rtx (word_mode);
2013 carry_out = emit_store_flag_force (carry_out,
2014 (binoptab == add_optab
2015 ? LT : GT),
2016 x, op0_piece,
2017 word_mode, 1, normalizep);
2018 }
2019
2020 if (i > 0)
2021 {
2022 rtx newx;
2023
2024 /* Add/subtract previous carry to main result. */
2025 newx = expand_binop (word_mode,
2026 normalizep == 1 ? binoptab : otheroptab,
2027 x, carry_in,
2028 NULL_RTX, 1, next_methods);
2029
2030 if (i + 1 < nwords)
2031 {
2032 /* Get out carry from adding/subtracting carry in. */
2033 rtx carry_tmp = gen_reg_rtx (word_mode);
2034 carry_tmp = emit_store_flag_force (carry_tmp,
2035 (binoptab == add_optab
2036 ? LT : GT),
2037 newx, x,
2038 word_mode, 1, normalizep);
2039
2040 /* Logical-ior the two poss. carry together. */
2041 carry_out = expand_binop (word_mode, ior_optab,
2042 carry_out, carry_tmp,
2043 carry_out, 0, next_methods);
2044 if (carry_out == 0)
2045 break;
2046 }
2047 emit_move_insn (target_piece, newx);
2048 }
2049 else
2050 {
2051 if (x != target_piece)
2052 emit_move_insn (target_piece, x);
2053 }
2054
2055 carry_in = carry_out;
2056 }
2057
2058 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
2059 {
2060 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
2061 || ! rtx_equal_p (target, xtarget))
2062 {
2063 rtx_insn *temp = emit_move_insn (target, xtarget);
2064
2065 set_dst_reg_note (temp, REG_EQUAL,
2066 gen_rtx_fmt_ee (optab_to_code (binoptab),
2067 int_mode, copy_rtx (xop0),
2068 copy_rtx (xop1)),
2069 target);
2070 }
2071 else
2072 target = xtarget;
2073
2074 return target;
2075 }
2076
2077 else
2078 delete_insns_since (last);
2079 }
2080
2081 /* Attempt to synthesize double word multiplies using a sequence of word
2082 mode multiplications. We first attempt to generate a sequence using a
2083 more efficient unsigned widening multiply, and if that fails we then
2084 try using a signed widening multiply. */
2085
2086 if (binoptab == smul_optab
2087 && is_int_mode (mode, &int_mode)
2088 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2089 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2090 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2091 {
2092 rtx product = NULL_RTX;
2093 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
2094 != CODE_FOR_nothing)
2095 {
2096 product = expand_doubleword_mult (int_mode, op0, op1, target,
2097 true, methods);
2098 if (!product)
2099 delete_insns_since (last);
2100 }
2101
2102 if (product == NULL_RTX
2103 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
2104 != CODE_FOR_nothing))
2105 {
2106 product = expand_doubleword_mult (int_mode, op0, op1, target,
2107 false, methods);
2108 if (!product)
2109 delete_insns_since (last);
2110 }
2111
2112 if (product != NULL_RTX)
2113 {
2114 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2115 {
2116 rtx_insn *move = emit_move_insn (target ? target : product,
2117 product);
2118 set_dst_reg_note (move,
2119 REG_EQUAL,
2120 gen_rtx_fmt_ee (MULT, int_mode,
2121 copy_rtx (op0),
2122 copy_rtx (op1)),
2123 target ? target : product);
2124 }
2125 return product;
2126 }
2127 }
2128
2129 /* Attempt to synthetize double word modulo by constant divisor. */
2130 if ((binoptab == umod_optab
2131 || binoptab == smod_optab
2132 || binoptab == udiv_optab
2133 || binoptab == sdiv_optab)
2134 && optimize
2135 && CONST_INT_P (op1)
2136 && is_int_mode (mode, &int_mode)
2137 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2138 && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
2139 ? udivmod_optab : sdivmod_optab,
2140 int_mode) == CODE_FOR_nothing
2141 && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
2142 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
2143 && optimize_insn_for_speed_p ())
2144 {
2145 rtx res = NULL_RTX;
2146 if ((binoptab == umod_optab || binoptab == smod_optab)
2147 && (INTVAL (op1) & 1) == 0)
2148 res = expand_doubleword_mod (int_mode, op0, op1,
2149 binoptab == umod_optab);
2150 else
2151 {
2152 rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
2153 binoptab == umod_optab
2154 || binoptab == udiv_optab);
2155 if (quot == NULL_RTX)
2156 res = NULL_RTX;
2157 else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2158 res = quot;
2159 }
2160 if (res != NULL_RTX)
2161 {
2162 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2163 {
2164 rtx_insn *move = emit_move_insn (target ? target : res,
2165 res);
2166 set_dst_reg_note (move, REG_EQUAL,
2167 gen_rtx_fmt_ee (optab_to_code (binoptab),
2168 int_mode, copy_rtx (op0), op1),
2169 target ? target : res);
2170 }
2171 return res;
2172 }
2173 else
2174 delete_insns_since (last);
2175 }
2176
2177 /* It can't be open-coded in this mode.
2178 Use a library call if one is available and caller says that's ok. */
2179
2180 libfunc = optab_libfunc (binoptab, mode);
2181 if (libfunc
2182 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2183 {
2184 rtx_insn *insns;
2185 rtx op1x = op1;
2186 machine_mode op1_mode = mode;
2187 rtx value;
2188
2189 start_sequence ();
2190
2191 if (shift_optab_p (binoptab))
2192 {
2193 op1_mode = targetm.libgcc_shift_count_mode ();
2194 /* Specify unsigned here,
2195 since negative shift counts are meaningless. */
2196 op1x = convert_to_mode (op1_mode, op1, 1);
2197 }
2198
2199 if (GET_MODE (op0) != VOIDmode
2200 && GET_MODE (op0) != mode)
2201 op0 = convert_to_mode (mode, op0, unsignedp);
2202
2203 /* Pass 1 for NO_QUEUE so we don't lose any increments
2204 if the libcall is cse'd or moved. */
2205 value = emit_library_call_value (libfunc,
2206 NULL_RTX, LCT_CONST, mode,
2207 op0, mode, op1x, op1_mode);
2208
2209 insns = get_insns ();
2210 end_sequence ();
2211
2212 bool trapv = trapv_binoptab_p (binoptab);
2213 target = gen_reg_rtx (mode);
2214 emit_libcall_block_1 (insns, target, value,
2215 trapv ? NULL_RTX
2216 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2217 mode, op0, op1), trapv);
2218
2219 return target;
2220 }
2221
2222 delete_insns_since (last);
2223
2224 /* It can't be done in this mode. Can we do it in a wider mode? */
2225
2226 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2227 || methods == OPTAB_MUST_WIDEN))
2228 {
2229 /* Caller says, don't even try. */
2230 delete_insns_since (entry_last);
2231 return 0;
2232 }
2233
2234 /* Compute the value of METHODS to pass to recursive calls.
2235 Don't allow widening to be tried recursively. */
2236
2237 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2238
2239 /* Look for a wider mode of the same class for which it appears we can do
2240 the operation. */
2241
2242 if (CLASS_HAS_WIDER_MODES_P (mclass))
2243 {
2244 /* This code doesn't make sense for conversion optabs, since we
2245 wouldn't then want to extend the operands to be the same size
2246 as the result. */
2247 gcc_assert (!convert_optab_p (binoptab));
2248 FOR_EACH_WIDER_MODE (wider_mode, mode)
2249 {
2250 if (optab_handler (binoptab, wider_mode)
2251 || (methods == OPTAB_LIB
2252 && optab_libfunc (binoptab, wider_mode)))
2253 {
2254 rtx xop0 = op0, xop1 = op1;
2255 int no_extend = 0;
2256
2257 /* For certain integer operations, we need not actually extend
2258 the narrow operands, as long as we will truncate
2259 the results to the same narrowness. */
2260
2261 if ((binoptab == ior_optab || binoptab == and_optab
2262 || binoptab == xor_optab
2263 || binoptab == add_optab || binoptab == sub_optab
2264 || binoptab == smul_optab || binoptab == ashl_optab)
2265 && mclass == MODE_INT)
2266 no_extend = 1;
2267
2268 xop0 = widen_operand (xop0, wider_mode, mode,
2269 unsignedp, no_extend);
2270
2271 /* The second operand of a shift must always be extended. */
2272 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2273 no_extend && binoptab != ashl_optab);
2274
2275 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2276 unsignedp, methods);
2277 if (temp)
2278 {
2279 if (mclass != MODE_INT
2280 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2281 {
2282 if (target == 0)
2283 target = gen_reg_rtx (mode);
2284 convert_move (target, temp, 0);
2285 return target;
2286 }
2287 else
2288 return gen_lowpart (mode, temp);
2289 }
2290 else
2291 delete_insns_since (last);
2292 }
2293 }
2294 }
2295
2296 delete_insns_since (entry_last);
2297 return 0;
2298 }
2299
2300 /* Expand a binary operator which has both signed and unsigned forms.
2301 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2302 signed operations.
2303
2304 If we widen unsigned operands, we may use a signed wider operation instead
2305 of an unsigned wider operation, since the result would be the same. */
2306
2307 rtx
sign_expand_binop(machine_mode mode,optab uoptab,optab soptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)2308 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2309 rtx op0, rtx op1, rtx target, int unsignedp,
2310 enum optab_methods methods)
2311 {
2312 rtx temp;
2313 optab direct_optab = unsignedp ? uoptab : soptab;
2314 bool save_enable;
2315
2316 /* Do it without widening, if possible. */
2317 temp = expand_binop (mode, direct_optab, op0, op1, target,
2318 unsignedp, OPTAB_DIRECT);
2319 if (temp || methods == OPTAB_DIRECT)
2320 return temp;
2321
2322 /* Try widening to a signed int. Disable any direct use of any
2323 signed insn in the current mode. */
2324 save_enable = swap_optab_enable (soptab, mode, false);
2325
2326 temp = expand_binop (mode, soptab, op0, op1, target,
2327 unsignedp, OPTAB_WIDEN);
2328
2329 /* For unsigned operands, try widening to an unsigned int. */
2330 if (!temp && unsignedp)
2331 temp = expand_binop (mode, uoptab, op0, op1, target,
2332 unsignedp, OPTAB_WIDEN);
2333 if (temp || methods == OPTAB_WIDEN)
2334 goto egress;
2335
2336 /* Use the right width libcall if that exists. */
2337 temp = expand_binop (mode, direct_optab, op0, op1, target,
2338 unsignedp, OPTAB_LIB);
2339 if (temp || methods == OPTAB_LIB)
2340 goto egress;
2341
2342 /* Must widen and use a libcall, use either signed or unsigned. */
2343 temp = expand_binop (mode, soptab, op0, op1, target,
2344 unsignedp, methods);
2345 if (!temp && unsignedp)
2346 temp = expand_binop (mode, uoptab, op0, op1, target,
2347 unsignedp, methods);
2348
2349 egress:
2350 /* Undo the fiddling above. */
2351 if (save_enable)
2352 swap_optab_enable (soptab, mode, true);
2353 return temp;
2354 }
2355
2356 /* Generate code to perform an operation specified by UNOPPTAB
2357 on operand OP0, with two results to TARG0 and TARG1.
2358 We assume that the order of the operands for the instruction
2359 is TARG0, TARG1, OP0.
2360
2361 Either TARG0 or TARG1 may be zero, but what that means is that
2362 the result is not actually wanted. We will generate it into
2363 a dummy pseudo-reg and discard it. They may not both be zero.
2364
2365 Returns 1 if this operation can be performed; 0 if not. */
2366
2367 int
expand_twoval_unop(optab unoptab,rtx op0,rtx targ0,rtx targ1,int unsignedp)2368 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2369 int unsignedp)
2370 {
2371 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2372 enum mode_class mclass;
2373 machine_mode wider_mode;
2374 rtx_insn *entry_last = get_last_insn ();
2375 rtx_insn *last;
2376
2377 mclass = GET_MODE_CLASS (mode);
2378
2379 if (!targ0)
2380 targ0 = gen_reg_rtx (mode);
2381 if (!targ1)
2382 targ1 = gen_reg_rtx (mode);
2383
2384 /* Record where to go back to if we fail. */
2385 last = get_last_insn ();
2386
2387 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2388 {
2389 class expand_operand ops[3];
2390 enum insn_code icode = optab_handler (unoptab, mode);
2391
2392 create_fixed_operand (&ops[0], targ0);
2393 create_fixed_operand (&ops[1], targ1);
2394 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2395 if (maybe_expand_insn (icode, 3, ops))
2396 return 1;
2397 }
2398
2399 /* It can't be done in this mode. Can we do it in a wider mode? */
2400
2401 if (CLASS_HAS_WIDER_MODES_P (mclass))
2402 {
2403 FOR_EACH_WIDER_MODE (wider_mode, mode)
2404 {
2405 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2406 {
2407 rtx t0 = gen_reg_rtx (wider_mode);
2408 rtx t1 = gen_reg_rtx (wider_mode);
2409 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2410
2411 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2412 {
2413 convert_move (targ0, t0, unsignedp);
2414 convert_move (targ1, t1, unsignedp);
2415 return 1;
2416 }
2417 else
2418 delete_insns_since (last);
2419 }
2420 }
2421 }
2422
2423 delete_insns_since (entry_last);
2424 return 0;
2425 }
2426
2427 /* Generate code to perform an operation specified by BINOPTAB
2428 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2429 We assume that the order of the operands for the instruction
2430 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2431 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2432
2433 Either TARG0 or TARG1 may be zero, but what that means is that
2434 the result is not actually wanted. We will generate it into
2435 a dummy pseudo-reg and discard it. They may not both be zero.
2436
2437 Returns 1 if this operation can be performed; 0 if not. */
2438
2439 int
expand_twoval_binop(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,int unsignedp)2440 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2441 int unsignedp)
2442 {
2443 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2444 enum mode_class mclass;
2445 machine_mode wider_mode;
2446 rtx_insn *entry_last = get_last_insn ();
2447 rtx_insn *last;
2448
2449 mclass = GET_MODE_CLASS (mode);
2450
2451 if (!targ0)
2452 targ0 = gen_reg_rtx (mode);
2453 if (!targ1)
2454 targ1 = gen_reg_rtx (mode);
2455
2456 /* Record where to go back to if we fail. */
2457 last = get_last_insn ();
2458
2459 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2460 {
2461 class expand_operand ops[4];
2462 enum insn_code icode = optab_handler (binoptab, mode);
2463 machine_mode mode0 = insn_data[icode].operand[1].mode;
2464 machine_mode mode1 = insn_data[icode].operand[2].mode;
2465 rtx xop0 = op0, xop1 = op1;
2466
2467 /* If we are optimizing, force expensive constants into a register. */
2468 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2469 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2470
2471 create_fixed_operand (&ops[0], targ0);
2472 create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2473 create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2474 create_fixed_operand (&ops[3], targ1);
2475 if (maybe_expand_insn (icode, 4, ops))
2476 return 1;
2477 delete_insns_since (last);
2478 }
2479
2480 /* It can't be done in this mode. Can we do it in a wider mode? */
2481
2482 if (CLASS_HAS_WIDER_MODES_P (mclass))
2483 {
2484 FOR_EACH_WIDER_MODE (wider_mode, mode)
2485 {
2486 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2487 {
2488 rtx t0 = gen_reg_rtx (wider_mode);
2489 rtx t1 = gen_reg_rtx (wider_mode);
2490 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2491 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2492
2493 if (expand_twoval_binop (binoptab, cop0, cop1,
2494 t0, t1, unsignedp))
2495 {
2496 convert_move (targ0, t0, unsignedp);
2497 convert_move (targ1, t1, unsignedp);
2498 return 1;
2499 }
2500 else
2501 delete_insns_since (last);
2502 }
2503 }
2504 }
2505
2506 delete_insns_since (entry_last);
2507 return 0;
2508 }
2509
2510 /* Expand the two-valued library call indicated by BINOPTAB, but
2511 preserve only one of the values. If TARG0 is non-NULL, the first
2512 value is placed into TARG0; otherwise the second value is placed
2513 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2514 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2515 This routine assumes that the value returned by the library call is
2516 as if the return value was of an integral mode twice as wide as the
2517 mode of OP0. Returns 1 if the call was successful. */
2518
2519 bool
expand_twoval_binop_libfunc(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,enum rtx_code code)2520 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2521 rtx targ0, rtx targ1, enum rtx_code code)
2522 {
2523 machine_mode mode;
2524 machine_mode libval_mode;
2525 rtx libval;
2526 rtx_insn *insns;
2527 rtx libfunc;
2528
2529 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2530 gcc_assert (!targ0 != !targ1);
2531
2532 mode = GET_MODE (op0);
2533 libfunc = optab_libfunc (binoptab, mode);
2534 if (!libfunc)
2535 return false;
2536
2537 /* The value returned by the library function will have twice as
2538 many bits as the nominal MODE. */
2539 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2540 start_sequence ();
2541 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2542 libval_mode,
2543 op0, mode,
2544 op1, mode);
2545 /* Get the part of VAL containing the value that we want. */
2546 libval = simplify_gen_subreg (mode, libval, libval_mode,
2547 targ0 ? 0 : GET_MODE_SIZE (mode));
2548 insns = get_insns ();
2549 end_sequence ();
2550 /* Move the into the desired location. */
2551 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2552 gen_rtx_fmt_ee (code, mode, op0, op1));
2553
2554 return true;
2555 }
2556
2557
2558 /* Wrapper around expand_unop which takes an rtx code to specify
2559 the operation to perform, not an optab pointer. All other
2560 arguments are the same. */
2561 rtx
expand_simple_unop(machine_mode mode,enum rtx_code code,rtx op0,rtx target,int unsignedp)2562 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2563 rtx target, int unsignedp)
2564 {
2565 optab unop = code_to_optab (code);
2566 gcc_assert (unop);
2567
2568 return expand_unop (mode, unop, op0, target, unsignedp);
2569 }
2570
2571 /* Try calculating
2572 (clz:narrow x)
2573 as
2574 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2575
2576 A similar operation can be used for clrsb. UNOPTAB says which operation
2577 we are trying to expand. */
2578 static rtx
widen_leading(scalar_int_mode mode,rtx op0,rtx target,optab unoptab)2579 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2580 {
2581 opt_scalar_int_mode wider_mode_iter;
2582 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2583 {
2584 scalar_int_mode wider_mode = wider_mode_iter.require ();
2585 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2586 {
2587 rtx xop0, temp;
2588 rtx_insn *last;
2589
2590 last = get_last_insn ();
2591
2592 if (target == 0)
2593 target = gen_reg_rtx (mode);
2594 xop0 = widen_operand (op0, wider_mode, mode,
2595 unoptab != clrsb_optab, false);
2596 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2597 unoptab != clrsb_optab);
2598 if (temp != 0)
2599 temp = expand_binop
2600 (wider_mode, sub_optab, temp,
2601 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2602 - GET_MODE_PRECISION (mode),
2603 wider_mode),
2604 target, true, OPTAB_DIRECT);
2605 if (temp == 0)
2606 delete_insns_since (last);
2607
2608 return temp;
2609 }
2610 }
2611 return 0;
2612 }
2613
2614 /* Attempt to emit (clrsb:mode op0) as
2615 (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2616 (const_int -1))
2617 if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2618 or as
2619 (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2620 (ashr:mode op0 (const_int prec-1)))
2621 (const_int 1)))
2622 otherwise. */
2623
2624 static rtx
expand_clrsb_using_clz(scalar_int_mode mode,rtx op0,rtx target)2625 expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2626 {
2627 if (optimize_insn_for_size_p ()
2628 || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2629 return NULL_RTX;
2630
2631 start_sequence ();
2632 HOST_WIDE_INT val = 0;
2633 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2634 || val != GET_MODE_PRECISION (mode))
2635 val = 0;
2636 else
2637 val = 1;
2638
2639 rtx temp2 = op0;
2640 if (!val)
2641 {
2642 temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
2643 NULL_RTX, 0, OPTAB_DIRECT);
2644 if (!temp2)
2645 {
2646 fail:
2647 end_sequence ();
2648 return NULL_RTX;
2649 }
2650 }
2651
2652 rtx temp = expand_binop (mode, ashr_optab, op0,
2653 GEN_INT (GET_MODE_PRECISION (mode) - 1),
2654 NULL_RTX, 0, OPTAB_DIRECT);
2655 if (!temp)
2656 goto fail;
2657
2658 temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
2659 OPTAB_DIRECT);
2660 if (!temp)
2661 goto fail;
2662
2663 if (!val)
2664 {
2665 temp = expand_binop (mode, ior_optab, temp, const1_rtx,
2666 NULL_RTX, 0, OPTAB_DIRECT);
2667 if (!temp)
2668 goto fail;
2669 }
2670 temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2671 true);
2672 if (!temp)
2673 goto fail;
2674 if (val)
2675 {
2676 temp = expand_binop (mode, add_optab, temp, constm1_rtx,
2677 target, 0, OPTAB_DIRECT);
2678 if (!temp)
2679 goto fail;
2680 }
2681
2682 rtx_insn *seq = get_insns ();
2683 end_sequence ();
2684
2685 add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
2686 emit_insn (seq);
2687 return temp;
2688 }
2689
2690 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2691 quantities, choosing which based on whether the high word is nonzero. */
2692 static rtx
expand_doubleword_clz(scalar_int_mode mode,rtx op0,rtx target)2693 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2694 {
2695 rtx xop0 = force_reg (mode, op0);
2696 rtx subhi = gen_highpart (word_mode, xop0);
2697 rtx sublo = gen_lowpart (word_mode, xop0);
2698 rtx_code_label *hi0_label = gen_label_rtx ();
2699 rtx_code_label *after_label = gen_label_rtx ();
2700 rtx_insn *seq;
2701 rtx temp, result;
2702
2703 /* If we were not given a target, use a word_mode register, not a
2704 'mode' register. The result will fit, and nobody is expecting
2705 anything bigger (the return type of __builtin_clz* is int). */
2706 if (!target)
2707 target = gen_reg_rtx (word_mode);
2708
2709 /* In any case, write to a word_mode scratch in both branches of the
2710 conditional, so we can ensure there is a single move insn setting
2711 'target' to tag a REG_EQUAL note on. */
2712 result = gen_reg_rtx (word_mode);
2713
2714 start_sequence ();
2715
2716 /* If the high word is not equal to zero,
2717 then clz of the full value is clz of the high word. */
2718 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2719 word_mode, true, hi0_label);
2720
2721 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2722 if (!temp)
2723 goto fail;
2724
2725 if (temp != result)
2726 convert_move (result, temp, true);
2727
2728 emit_jump_insn (targetm.gen_jump (after_label));
2729 emit_barrier ();
2730
2731 /* Else clz of the full value is clz of the low word plus the number
2732 of bits in the high word. */
2733 emit_label (hi0_label);
2734
2735 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2736 if (!temp)
2737 goto fail;
2738 temp = expand_binop (word_mode, add_optab, temp,
2739 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2740 result, true, OPTAB_DIRECT);
2741 if (!temp)
2742 goto fail;
2743 if (temp != result)
2744 convert_move (result, temp, true);
2745
2746 emit_label (after_label);
2747 convert_move (target, result, true);
2748
2749 seq = get_insns ();
2750 end_sequence ();
2751
2752 add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode);
2753 emit_insn (seq);
2754 return target;
2755
2756 fail:
2757 end_sequence ();
2758 return 0;
2759 }
2760
2761 /* Try calculating popcount of a double-word quantity as two popcount's of
2762 word-sized quantities and summing up the results. */
2763 static rtx
expand_doubleword_popcount(scalar_int_mode mode,rtx op0,rtx target)2764 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2765 {
2766 rtx t0, t1, t;
2767 rtx_insn *seq;
2768
2769 start_sequence ();
2770
2771 t0 = expand_unop_direct (word_mode, popcount_optab,
2772 operand_subword_force (op0, 0, mode), NULL_RTX,
2773 true);
2774 t1 = expand_unop_direct (word_mode, popcount_optab,
2775 operand_subword_force (op0, 1, mode), NULL_RTX,
2776 true);
2777 if (!t0 || !t1)
2778 {
2779 end_sequence ();
2780 return NULL_RTX;
2781 }
2782
2783 /* If we were not given a target, use a word_mode register, not a
2784 'mode' register. The result will fit, and nobody is expecting
2785 anything bigger (the return type of __builtin_popcount* is int). */
2786 if (!target)
2787 target = gen_reg_rtx (word_mode);
2788
2789 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2790
2791 seq = get_insns ();
2792 end_sequence ();
2793
2794 add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2795 emit_insn (seq);
2796 return t;
2797 }
2798
2799 /* Try calculating
2800 (parity:wide x)
2801 as
2802 (parity:narrow (low (x) ^ high (x))) */
2803 static rtx
expand_doubleword_parity(scalar_int_mode mode,rtx op0,rtx target)2804 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2805 {
2806 rtx t = expand_binop (word_mode, xor_optab,
2807 operand_subword_force (op0, 0, mode),
2808 operand_subword_force (op0, 1, mode),
2809 NULL_RTX, 0, OPTAB_DIRECT);
2810 return expand_unop (word_mode, parity_optab, t, target, true);
2811 }
2812
2813 /* Try calculating
2814 (bswap:narrow x)
2815 as
2816 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2817 static rtx
widen_bswap(scalar_int_mode mode,rtx op0,rtx target)2818 widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2819 {
2820 rtx x;
2821 rtx_insn *last;
2822 opt_scalar_int_mode wider_mode_iter;
2823
2824 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2825 if (optab_handler (bswap_optab, wider_mode_iter.require ())
2826 != CODE_FOR_nothing)
2827 break;
2828
2829 if (!wider_mode_iter.exists ())
2830 return NULL_RTX;
2831
2832 scalar_int_mode wider_mode = wider_mode_iter.require ();
2833 last = get_last_insn ();
2834
2835 x = widen_operand (op0, wider_mode, mode, true, true);
2836 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2837
2838 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2839 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2840 if (x != 0)
2841 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2842 GET_MODE_BITSIZE (wider_mode)
2843 - GET_MODE_BITSIZE (mode),
2844 NULL_RTX, true);
2845
2846 if (x != 0)
2847 {
2848 if (target == 0)
2849 target = gen_reg_rtx (mode);
2850 emit_move_insn (target, gen_lowpart (mode, x));
2851 }
2852 else
2853 delete_insns_since (last);
2854
2855 return target;
2856 }
2857
2858 /* Try calculating bswap as two bswaps of two word-sized operands. */
2859
2860 static rtx
expand_doubleword_bswap(machine_mode mode,rtx op,rtx target)2861 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2862 {
2863 rtx t0, t1;
2864
2865 t1 = expand_unop (word_mode, bswap_optab,
2866 operand_subword_force (op, 0, mode), NULL_RTX, true);
2867 t0 = expand_unop (word_mode, bswap_optab,
2868 operand_subword_force (op, 1, mode), NULL_RTX, true);
2869
2870 if (target == 0 || !valid_multiword_target_p (target))
2871 target = gen_reg_rtx (mode);
2872 if (REG_P (target))
2873 emit_clobber (target);
2874 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2875 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2876
2877 return target;
2878 }
2879
2880 /* Try calculating (parity x) as (and (popcount x) 1), where
2881 popcount can also be done in a wider mode. */
2882 static rtx
expand_parity(scalar_int_mode mode,rtx op0,rtx target)2883 expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2884 {
2885 enum mode_class mclass = GET_MODE_CLASS (mode);
2886 opt_scalar_int_mode wider_mode_iter;
2887 FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2888 {
2889 scalar_int_mode wider_mode = wider_mode_iter.require ();
2890 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2891 {
2892 rtx xop0, temp;
2893 rtx_insn *last;
2894
2895 last = get_last_insn ();
2896
2897 if (target == 0 || GET_MODE (target) != wider_mode)
2898 target = gen_reg_rtx (wider_mode);
2899
2900 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2901 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2902 true);
2903 if (temp != 0)
2904 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2905 target, true, OPTAB_DIRECT);
2906
2907 if (temp)
2908 {
2909 if (mclass != MODE_INT
2910 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2911 return convert_to_mode (mode, temp, 0);
2912 else
2913 return gen_lowpart (mode, temp);
2914 }
2915 else
2916 delete_insns_since (last);
2917 }
2918 }
2919 return 0;
2920 }
2921
2922 /* Try calculating ctz(x) as K - clz(x & -x) ,
2923 where K is GET_MODE_PRECISION(mode) - 1.
2924
2925 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2926 don't have to worry about what the hardware does in that case. (If
2927 the clz instruction produces the usual value at 0, which is K, the
2928 result of this code sequence will be -1; expand_ffs, below, relies
2929 on this. It might be nice to have it be K instead, for consistency
2930 with the (very few) processors that provide a ctz with a defined
2931 value, but that would take one more instruction, and it would be
2932 less convenient for expand_ffs anyway. */
2933
2934 static rtx
expand_ctz(scalar_int_mode mode,rtx op0,rtx target)2935 expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2936 {
2937 rtx_insn *seq;
2938 rtx temp;
2939
2940 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2941 return 0;
2942
2943 start_sequence ();
2944
2945 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2946 if (temp)
2947 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2948 true, OPTAB_DIRECT);
2949 if (temp)
2950 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2951 if (temp)
2952 temp = expand_binop (mode, sub_optab,
2953 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2954 temp, target,
2955 true, OPTAB_DIRECT);
2956 if (temp == 0)
2957 {
2958 end_sequence ();
2959 return 0;
2960 }
2961
2962 seq = get_insns ();
2963 end_sequence ();
2964
2965 add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
2966 emit_insn (seq);
2967 return temp;
2968 }
2969
2970
2971 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2972 else with the sequence used by expand_clz.
2973
2974 The ffs builtin promises to return zero for a zero value and ctz/clz
2975 may have an undefined value in that case. If they do not give us a
2976 convenient value, we have to generate a test and branch. */
2977 static rtx
expand_ffs(scalar_int_mode mode,rtx op0,rtx target)2978 expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2979 {
2980 HOST_WIDE_INT val = 0;
2981 bool defined_at_zero = false;
2982 rtx temp;
2983 rtx_insn *seq;
2984
2985 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2986 {
2987 start_sequence ();
2988
2989 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2990 if (!temp)
2991 goto fail;
2992
2993 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2994 }
2995 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2996 {
2997 start_sequence ();
2998 temp = expand_ctz (mode, op0, 0);
2999 if (!temp)
3000 goto fail;
3001
3002 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3003 {
3004 defined_at_zero = true;
3005 val = (GET_MODE_PRECISION (mode) - 1) - val;
3006 }
3007 }
3008 else
3009 return 0;
3010
3011 if (defined_at_zero && val == -1)
3012 /* No correction needed at zero. */;
3013 else
3014 {
3015 /* We don't try to do anything clever with the situation found
3016 on some processors (eg Alpha) where ctz(0:mode) ==
3017 bitsize(mode). If someone can think of a way to send N to -1
3018 and leave alone all values in the range 0..N-1 (where N is a
3019 power of two), cheaper than this test-and-branch, please add it.
3020
3021 The test-and-branch is done after the operation itself, in case
3022 the operation sets condition codes that can be recycled for this.
3023 (This is true on i386, for instance.) */
3024
3025 rtx_code_label *nonzero_label = gen_label_rtx ();
3026 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3027 mode, true, nonzero_label);
3028
3029 convert_move (temp, GEN_INT (-1), false);
3030 emit_label (nonzero_label);
3031 }
3032
3033 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3034 to produce a value in the range 0..bitsize. */
3035 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
3036 target, false, OPTAB_DIRECT);
3037 if (!temp)
3038 goto fail;
3039
3040 seq = get_insns ();
3041 end_sequence ();
3042
3043 add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
3044 emit_insn (seq);
3045 return temp;
3046
3047 fail:
3048 end_sequence ();
3049 return 0;
3050 }
3051
3052 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
3053 conditions, VAL may already be a SUBREG against which we cannot generate
3054 a further SUBREG. In this case, we expect forcing the value into a
3055 register will work around the situation. */
3056
3057 static rtx
lowpart_subreg_maybe_copy(machine_mode omode,rtx val,machine_mode imode)3058 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
3059 machine_mode imode)
3060 {
3061 rtx ret;
3062 ret = lowpart_subreg (omode, val, imode);
3063 if (ret == NULL)
3064 {
3065 val = force_reg (imode, val);
3066 ret = lowpart_subreg (omode, val, imode);
3067 gcc_assert (ret != NULL);
3068 }
3069 return ret;
3070 }
3071
3072 /* Expand a floating point absolute value or negation operation via a
3073 logical operation on the sign bit. */
3074
3075 static rtx
expand_absneg_bit(enum rtx_code code,scalar_float_mode mode,rtx op0,rtx target)3076 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
3077 rtx op0, rtx target)
3078 {
3079 const struct real_format *fmt;
3080 int bitpos, word, nwords, i;
3081 scalar_int_mode imode;
3082 rtx temp;
3083 rtx_insn *insns;
3084
3085 /* The format has to have a simple sign bit. */
3086 fmt = REAL_MODE_FORMAT (mode);
3087 if (fmt == NULL)
3088 return NULL_RTX;
3089
3090 bitpos = fmt->signbit_rw;
3091 if (bitpos < 0)
3092 return NULL_RTX;
3093
3094 /* Don't create negative zeros if the format doesn't support them. */
3095 if (code == NEG && !fmt->has_signed_zero)
3096 return NULL_RTX;
3097
3098 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3099 {
3100 if (!int_mode_for_mode (mode).exists (&imode))
3101 return NULL_RTX;
3102 word = 0;
3103 nwords = 1;
3104 }
3105 else
3106 {
3107 imode = word_mode;
3108
3109 if (FLOAT_WORDS_BIG_ENDIAN)
3110 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3111 else
3112 word = bitpos / BITS_PER_WORD;
3113 bitpos = bitpos % BITS_PER_WORD;
3114 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3115 }
3116
3117 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3118 if (code == ABS)
3119 mask = ~mask;
3120
3121 if (target == 0
3122 || target == op0
3123 || reg_overlap_mentioned_p (target, op0)
3124 || (nwords > 1 && !valid_multiword_target_p (target)))
3125 target = gen_reg_rtx (mode);
3126
3127 if (nwords > 1)
3128 {
3129 start_sequence ();
3130
3131 for (i = 0; i < nwords; ++i)
3132 {
3133 rtx targ_piece = operand_subword (target, i, 1, mode);
3134 rtx op0_piece = operand_subword_force (op0, i, mode);
3135
3136 if (i == word)
3137 {
3138 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3139 op0_piece,
3140 immed_wide_int_const (mask, imode),
3141 targ_piece, 1, OPTAB_LIB_WIDEN);
3142 if (temp != targ_piece)
3143 emit_move_insn (targ_piece, temp);
3144 }
3145 else
3146 emit_move_insn (targ_piece, op0_piece);
3147 }
3148
3149 insns = get_insns ();
3150 end_sequence ();
3151
3152 emit_insn (insns);
3153 }
3154 else
3155 {
3156 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3157 gen_lowpart (imode, op0),
3158 immed_wide_int_const (mask, imode),
3159 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3160 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3161
3162 set_dst_reg_note (get_last_insn (), REG_EQUAL,
3163 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3164 target);
3165 }
3166
3167 return target;
3168 }
3169
3170 /* As expand_unop, but will fail rather than attempt the operation in a
3171 different mode or with a libcall. */
3172 static rtx
expand_unop_direct(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)3173 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3174 int unsignedp)
3175 {
3176 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3177 {
3178 class expand_operand ops[2];
3179 enum insn_code icode = optab_handler (unoptab, mode);
3180 rtx_insn *last = get_last_insn ();
3181 rtx_insn *pat;
3182
3183 create_output_operand (&ops[0], target, mode);
3184 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3185 pat = maybe_gen_insn (icode, 2, ops);
3186 if (pat)
3187 {
3188 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3189 && ! add_equal_note (pat, ops[0].value,
3190 optab_to_code (unoptab),
3191 ops[1].value, NULL_RTX, mode))
3192 {
3193 delete_insns_since (last);
3194 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3195 }
3196
3197 emit_insn (pat);
3198
3199 return ops[0].value;
3200 }
3201 }
3202 return 0;
3203 }
3204
3205 /* Generate code to perform an operation specified by UNOPTAB
3206 on operand OP0, with result having machine-mode MODE.
3207
3208 UNSIGNEDP is for the case where we have to widen the operands
3209 to perform the operation. It says to use zero-extension.
3210
3211 If TARGET is nonzero, the value
3212 is generated there, if it is convenient to do so.
3213 In all cases an rtx is returned for the locus of the value;
3214 this may or may not be TARGET. */
3215
3216 rtx
expand_unop(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)3217 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3218 int unsignedp)
3219 {
3220 enum mode_class mclass = GET_MODE_CLASS (mode);
3221 machine_mode wider_mode;
3222 scalar_int_mode int_mode;
3223 scalar_float_mode float_mode;
3224 rtx temp;
3225 rtx libfunc;
3226
3227 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3228 if (temp)
3229 return temp;
3230
3231 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3232
3233 /* Widening (or narrowing) clz needs special treatment. */
3234 if (unoptab == clz_optab)
3235 {
3236 if (is_a <scalar_int_mode> (mode, &int_mode))
3237 {
3238 temp = widen_leading (int_mode, op0, target, unoptab);
3239 if (temp)
3240 return temp;
3241
3242 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3243 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3244 {
3245 temp = expand_doubleword_clz (int_mode, op0, target);
3246 if (temp)
3247 return temp;
3248 }
3249 }
3250
3251 goto try_libcall;
3252 }
3253
3254 if (unoptab == clrsb_optab)
3255 {
3256 if (is_a <scalar_int_mode> (mode, &int_mode))
3257 {
3258 temp = widen_leading (int_mode, op0, target, unoptab);
3259 if (temp)
3260 return temp;
3261 temp = expand_clrsb_using_clz (int_mode, op0, target);
3262 if (temp)
3263 return temp;
3264 }
3265 goto try_libcall;
3266 }
3267
3268 if (unoptab == popcount_optab
3269 && is_a <scalar_int_mode> (mode, &int_mode)
3270 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3271 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3272 && optimize_insn_for_speed_p ())
3273 {
3274 temp = expand_doubleword_popcount (int_mode, op0, target);
3275 if (temp)
3276 return temp;
3277 }
3278
3279 if (unoptab == parity_optab
3280 && is_a <scalar_int_mode> (mode, &int_mode)
3281 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3282 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3283 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
3284 && optimize_insn_for_speed_p ())
3285 {
3286 temp = expand_doubleword_parity (int_mode, op0, target);
3287 if (temp)
3288 return temp;
3289 }
3290
3291 /* Widening (or narrowing) bswap needs special treatment. */
3292 if (unoptab == bswap_optab)
3293 {
3294 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3295 or ROTATERT. First try these directly; if this fails, then try the
3296 obvious pair of shifts with allowed widening, as this will probably
3297 be always more efficient than the other fallback methods. */
3298 if (mode == HImode)
3299 {
3300 rtx_insn *last;
3301 rtx temp1, temp2;
3302
3303 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3304 {
3305 temp = expand_binop (mode, rotl_optab, op0,
3306 gen_int_shift_amount (mode, 8),
3307 target, unsignedp, OPTAB_DIRECT);
3308 if (temp)
3309 return temp;
3310 }
3311
3312 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3313 {
3314 temp = expand_binop (mode, rotr_optab, op0,
3315 gen_int_shift_amount (mode, 8),
3316 target, unsignedp, OPTAB_DIRECT);
3317 if (temp)
3318 return temp;
3319 }
3320
3321 last = get_last_insn ();
3322
3323 temp1 = expand_binop (mode, ashl_optab, op0,
3324 gen_int_shift_amount (mode, 8), NULL_RTX,
3325 unsignedp, OPTAB_WIDEN);
3326 temp2 = expand_binop (mode, lshr_optab, op0,
3327 gen_int_shift_amount (mode, 8), NULL_RTX,
3328 unsignedp, OPTAB_WIDEN);
3329 if (temp1 && temp2)
3330 {
3331 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3332 unsignedp, OPTAB_WIDEN);
3333 if (temp)
3334 return temp;
3335 }
3336
3337 delete_insns_since (last);
3338 }
3339
3340 if (is_a <scalar_int_mode> (mode, &int_mode))
3341 {
3342 temp = widen_bswap (int_mode, op0, target);
3343 if (temp)
3344 return temp;
3345
3346 /* We do not provide a 128-bit bswap in libgcc so force the use of
3347 a double bswap for 64-bit targets. */
3348 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3349 && (UNITS_PER_WORD == 8
3350 || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
3351 {
3352 temp = expand_doubleword_bswap (mode, op0, target);
3353 if (temp)
3354 return temp;
3355 }
3356 }
3357
3358 goto try_libcall;
3359 }
3360
3361 if (CLASS_HAS_WIDER_MODES_P (mclass))
3362 FOR_EACH_WIDER_MODE (wider_mode, mode)
3363 {
3364 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3365 {
3366 rtx xop0 = op0;
3367 rtx_insn *last = get_last_insn ();
3368
3369 /* For certain operations, we need not actually extend
3370 the narrow operand, as long as we will truncate the
3371 results to the same narrowness. */
3372
3373 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3374 (unoptab == neg_optab
3375 || unoptab == one_cmpl_optab)
3376 && mclass == MODE_INT);
3377
3378 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3379 unsignedp);
3380
3381 if (temp)
3382 {
3383 if (mclass != MODE_INT
3384 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3385 {
3386 if (target == 0)
3387 target = gen_reg_rtx (mode);
3388 convert_move (target, temp, 0);
3389 return target;
3390 }
3391 else
3392 return gen_lowpart (mode, temp);
3393 }
3394 else
3395 delete_insns_since (last);
3396 }
3397 }
3398
3399 /* These can be done a word at a time. */
3400 if (unoptab == one_cmpl_optab
3401 && is_int_mode (mode, &int_mode)
3402 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
3403 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3404 {
3405 int i;
3406 rtx_insn *insns;
3407
3408 if (target == 0
3409 || target == op0
3410 || reg_overlap_mentioned_p (target, op0)
3411 || !valid_multiword_target_p (target))
3412 target = gen_reg_rtx (int_mode);
3413
3414 start_sequence ();
3415
3416 /* Do the actual arithmetic. */
3417 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
3418 {
3419 rtx target_piece = operand_subword (target, i, 1, int_mode);
3420 rtx x = expand_unop (word_mode, unoptab,
3421 operand_subword_force (op0, i, int_mode),
3422 target_piece, unsignedp);
3423
3424 if (target_piece != x)
3425 emit_move_insn (target_piece, x);
3426 }
3427
3428 insns = get_insns ();
3429 end_sequence ();
3430
3431 emit_insn (insns);
3432 return target;
3433 }
3434
3435 /* Emit ~op0 as op0 ^ -1. */
3436 if (unoptab == one_cmpl_optab
3437 && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3438 && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
3439 {
3440 temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
3441 target, unsignedp, OPTAB_DIRECT);
3442 if (temp)
3443 return temp;
3444 }
3445
3446 if (optab_to_code (unoptab) == NEG)
3447 {
3448 /* Try negating floating point values by flipping the sign bit. */
3449 if (is_a <scalar_float_mode> (mode, &float_mode))
3450 {
3451 temp = expand_absneg_bit (NEG, float_mode, op0, target);
3452 if (temp)
3453 return temp;
3454 }
3455
3456 /* If there is no negation pattern, and we have no negative zero,
3457 try subtracting from zero. */
3458 if (!HONOR_SIGNED_ZEROS (mode))
3459 {
3460 temp = expand_binop (mode, (unoptab == negv_optab
3461 ? subv_optab : sub_optab),
3462 CONST0_RTX (mode), op0, target,
3463 unsignedp, OPTAB_DIRECT);
3464 if (temp)
3465 return temp;
3466 }
3467 }
3468
3469 /* Try calculating parity (x) as popcount (x) % 2. */
3470 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3471 {
3472 temp = expand_parity (int_mode, op0, target);
3473 if (temp)
3474 return temp;
3475 }
3476
3477 /* Try implementing ffs (x) in terms of clz (x). */
3478 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3479 {
3480 temp = expand_ffs (int_mode, op0, target);
3481 if (temp)
3482 return temp;
3483 }
3484
3485 /* Try implementing ctz (x) in terms of clz (x). */
3486 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3487 {
3488 temp = expand_ctz (int_mode, op0, target);
3489 if (temp)
3490 return temp;
3491 }
3492
3493 try_libcall:
3494 /* Now try a library call in this mode. */
3495 libfunc = optab_libfunc (unoptab, mode);
3496 if (libfunc)
3497 {
3498 rtx_insn *insns;
3499 rtx value;
3500 rtx eq_value;
3501 machine_mode outmode = mode;
3502
3503 /* All of these functions return small values. Thus we choose to
3504 have them return something that isn't a double-word. */
3505 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3506 || unoptab == clrsb_optab || unoptab == popcount_optab
3507 || unoptab == parity_optab)
3508 outmode
3509 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3510 optab_libfunc (unoptab, mode)));
3511
3512 start_sequence ();
3513
3514 /* Pass 1 for NO_QUEUE so we don't lose any increments
3515 if the libcall is cse'd or moved. */
3516 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3517 op0, mode);
3518 insns = get_insns ();
3519 end_sequence ();
3520
3521 target = gen_reg_rtx (outmode);
3522 bool trapv = trapv_unoptab_p (unoptab);
3523 if (trapv)
3524 eq_value = NULL_RTX;
3525 else
3526 {
3527 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3528 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3529 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3530 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3531 eq_value = simplify_gen_unary (ZERO_EXTEND,
3532 outmode, eq_value, mode);
3533 }
3534 emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3535
3536 return target;
3537 }
3538
3539 /* It can't be done in this mode. Can we do it in a wider mode? */
3540
3541 if (CLASS_HAS_WIDER_MODES_P (mclass))
3542 {
3543 FOR_EACH_WIDER_MODE (wider_mode, mode)
3544 {
3545 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3546 || optab_libfunc (unoptab, wider_mode))
3547 {
3548 rtx xop0 = op0;
3549 rtx_insn *last = get_last_insn ();
3550
3551 /* For certain operations, we need not actually extend
3552 the narrow operand, as long as we will truncate the
3553 results to the same narrowness. */
3554 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3555 (unoptab == neg_optab
3556 || unoptab == one_cmpl_optab
3557 || unoptab == bswap_optab)
3558 && mclass == MODE_INT);
3559
3560 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3561 unsignedp);
3562
3563 /* If we are generating clz using wider mode, adjust the
3564 result. Similarly for clrsb. */
3565 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3566 && temp != 0)
3567 {
3568 scalar_int_mode wider_int_mode
3569 = as_a <scalar_int_mode> (wider_mode);
3570 int_mode = as_a <scalar_int_mode> (mode);
3571 temp = expand_binop
3572 (wider_mode, sub_optab, temp,
3573 gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3574 - GET_MODE_PRECISION (int_mode),
3575 wider_int_mode),
3576 target, true, OPTAB_DIRECT);
3577 }
3578
3579 /* Likewise for bswap. */
3580 if (unoptab == bswap_optab && temp != 0)
3581 {
3582 scalar_int_mode wider_int_mode
3583 = as_a <scalar_int_mode> (wider_mode);
3584 int_mode = as_a <scalar_int_mode> (mode);
3585 gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3586 == GET_MODE_BITSIZE (wider_int_mode)
3587 && GET_MODE_PRECISION (int_mode)
3588 == GET_MODE_BITSIZE (int_mode));
3589
3590 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3591 GET_MODE_BITSIZE (wider_int_mode)
3592 - GET_MODE_BITSIZE (int_mode),
3593 NULL_RTX, true);
3594 }
3595
3596 if (temp)
3597 {
3598 if (mclass != MODE_INT)
3599 {
3600 if (target == 0)
3601 target = gen_reg_rtx (mode);
3602 convert_move (target, temp, 0);
3603 return target;
3604 }
3605 else
3606 return gen_lowpart (mode, temp);
3607 }
3608 else
3609 delete_insns_since (last);
3610 }
3611 }
3612 }
3613
3614 /* One final attempt at implementing negation via subtraction,
3615 this time allowing widening of the operand. */
3616 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3617 {
3618 rtx temp;
3619 temp = expand_binop (mode,
3620 unoptab == negv_optab ? subv_optab : sub_optab,
3621 CONST0_RTX (mode), op0,
3622 target, unsignedp, OPTAB_LIB_WIDEN);
3623 if (temp)
3624 return temp;
3625 }
3626
3627 return 0;
3628 }
3629
3630 /* Emit code to compute the absolute value of OP0, with result to
3631 TARGET if convenient. (TARGET may be 0.) The return value says
3632 where the result actually is to be found.
3633
3634 MODE is the mode of the operand; the mode of the result is
3635 different but can be deduced from MODE.
3636
3637 */
3638
3639 rtx
expand_abs_nojump(machine_mode mode,rtx op0,rtx target,int result_unsignedp)3640 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3641 int result_unsignedp)
3642 {
3643 rtx temp;
3644
3645 if (GET_MODE_CLASS (mode) != MODE_INT
3646 || ! flag_trapv)
3647 result_unsignedp = 1;
3648
3649 /* First try to do it with a special abs instruction. */
3650 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3651 op0, target, 0);
3652 if (temp != 0)
3653 return temp;
3654
3655 /* For floating point modes, try clearing the sign bit. */
3656 scalar_float_mode float_mode;
3657 if (is_a <scalar_float_mode> (mode, &float_mode))
3658 {
3659 temp = expand_absneg_bit (ABS, float_mode, op0, target);
3660 if (temp)
3661 return temp;
3662 }
3663
3664 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3665 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3666 && !HONOR_SIGNED_ZEROS (mode))
3667 {
3668 rtx_insn *last = get_last_insn ();
3669
3670 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3671 op0, NULL_RTX, 0);
3672 if (temp != 0)
3673 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3674 OPTAB_WIDEN);
3675
3676 if (temp != 0)
3677 return temp;
3678
3679 delete_insns_since (last);
3680 }
3681
3682 /* If this machine has expensive jumps, we can do integer absolute
3683 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3684 where W is the width of MODE. */
3685
3686 scalar_int_mode int_mode;
3687 if (is_int_mode (mode, &int_mode)
3688 && BRANCH_COST (optimize_insn_for_speed_p (),
3689 false) >= 2)
3690 {
3691 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3692 GET_MODE_PRECISION (int_mode) - 1,
3693 NULL_RTX, 0);
3694
3695 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3696 OPTAB_LIB_WIDEN);
3697 if (temp != 0)
3698 temp = expand_binop (int_mode,
3699 result_unsignedp ? sub_optab : subv_optab,
3700 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3701
3702 if (temp != 0)
3703 return temp;
3704 }
3705
3706 return NULL_RTX;
3707 }
3708
3709 rtx
expand_abs(machine_mode mode,rtx op0,rtx target,int result_unsignedp,int safe)3710 expand_abs (machine_mode mode, rtx op0, rtx target,
3711 int result_unsignedp, int safe)
3712 {
3713 rtx temp;
3714 rtx_code_label *op1;
3715
3716 if (GET_MODE_CLASS (mode) != MODE_INT
3717 || ! flag_trapv)
3718 result_unsignedp = 1;
3719
3720 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3721 if (temp != 0)
3722 return temp;
3723
3724 /* If that does not win, use conditional jump and negate. */
3725
3726 /* It is safe to use the target if it is the same
3727 as the source if this is also a pseudo register */
3728 if (op0 == target && REG_P (op0)
3729 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3730 safe = 1;
3731
3732 op1 = gen_label_rtx ();
3733 if (target == 0 || ! safe
3734 || GET_MODE (target) != mode
3735 || (MEM_P (target) && MEM_VOLATILE_P (target))
3736 || (REG_P (target)
3737 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3738 target = gen_reg_rtx (mode);
3739
3740 emit_move_insn (target, op0);
3741 NO_DEFER_POP;
3742
3743 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3744 NULL_RTX, NULL, op1,
3745 profile_probability::uninitialized ());
3746
3747 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3748 target, target, 0);
3749 if (op0 != target)
3750 emit_move_insn (target, op0);
3751 emit_label (op1);
3752 OK_DEFER_POP;
3753 return target;
3754 }
3755
3756 /* Emit code to compute the one's complement absolute value of OP0
3757 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3758 (TARGET may be NULL_RTX.) The return value says where the result
3759 actually is to be found.
3760
3761 MODE is the mode of the operand; the mode of the result is
3762 different but can be deduced from MODE. */
3763
3764 rtx
expand_one_cmpl_abs_nojump(machine_mode mode,rtx op0,rtx target)3765 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3766 {
3767 rtx temp;
3768
3769 /* Not applicable for floating point modes. */
3770 if (FLOAT_MODE_P (mode))
3771 return NULL_RTX;
3772
3773 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3774 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3775 {
3776 rtx_insn *last = get_last_insn ();
3777
3778 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3779 if (temp != 0)
3780 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3781 OPTAB_WIDEN);
3782
3783 if (temp != 0)
3784 return temp;
3785
3786 delete_insns_since (last);
3787 }
3788
3789 /* If this machine has expensive jumps, we can do one's complement
3790 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3791
3792 scalar_int_mode int_mode;
3793 if (is_int_mode (mode, &int_mode)
3794 && BRANCH_COST (optimize_insn_for_speed_p (),
3795 false) >= 2)
3796 {
3797 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3798 GET_MODE_PRECISION (int_mode) - 1,
3799 NULL_RTX, 0);
3800
3801 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3802 OPTAB_LIB_WIDEN);
3803
3804 if (temp != 0)
3805 return temp;
3806 }
3807
3808 return NULL_RTX;
3809 }
3810
3811 /* A subroutine of expand_copysign, perform the copysign operation using the
3812 abs and neg primitives advertised to exist on the target. The assumption
3813 is that we have a split register file, and leaving op0 in fp registers,
3814 and not playing with subregs so much, will help the register allocator. */
3815
3816 static rtx
expand_copysign_absneg(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3817 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3818 int bitpos, bool op0_is_abs)
3819 {
3820 scalar_int_mode imode;
3821 enum insn_code icode;
3822 rtx sign;
3823 rtx_code_label *label;
3824
3825 if (target == op1)
3826 target = NULL_RTX;
3827
3828 /* Check if the back end provides an insn that handles signbit for the
3829 argument's mode. */
3830 icode = optab_handler (signbit_optab, mode);
3831 if (icode != CODE_FOR_nothing)
3832 {
3833 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3834 sign = gen_reg_rtx (imode);
3835 emit_unop_insn (icode, sign, op1, UNKNOWN);
3836 }
3837 else
3838 {
3839 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3840 {
3841 if (!int_mode_for_mode (mode).exists (&imode))
3842 return NULL_RTX;
3843 op1 = gen_lowpart (imode, op1);
3844 }
3845 else
3846 {
3847 int word;
3848
3849 imode = word_mode;
3850 if (FLOAT_WORDS_BIG_ENDIAN)
3851 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3852 else
3853 word = bitpos / BITS_PER_WORD;
3854 bitpos = bitpos % BITS_PER_WORD;
3855 op1 = operand_subword_force (op1, word, mode);
3856 }
3857
3858 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3859 sign = expand_binop (imode, and_optab, op1,
3860 immed_wide_int_const (mask, imode),
3861 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3862 }
3863
3864 if (!op0_is_abs)
3865 {
3866 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3867 if (op0 == NULL)
3868 return NULL_RTX;
3869 target = op0;
3870 }
3871 else
3872 {
3873 if (target == NULL_RTX)
3874 target = copy_to_reg (op0);
3875 else
3876 emit_move_insn (target, op0);
3877 }
3878
3879 label = gen_label_rtx ();
3880 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3881
3882 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3883 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3884 else
3885 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3886 if (op0 != target)
3887 emit_move_insn (target, op0);
3888
3889 emit_label (label);
3890
3891 return target;
3892 }
3893
3894
3895 /* A subroutine of expand_copysign, perform the entire copysign operation
3896 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3897 is true if op0 is known to have its sign bit clear. */
3898
3899 static rtx
expand_copysign_bit(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3900 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3901 int bitpos, bool op0_is_abs)
3902 {
3903 scalar_int_mode imode;
3904 int word, nwords, i;
3905 rtx temp;
3906 rtx_insn *insns;
3907
3908 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3909 {
3910 if (!int_mode_for_mode (mode).exists (&imode))
3911 return NULL_RTX;
3912 word = 0;
3913 nwords = 1;
3914 }
3915 else
3916 {
3917 imode = word_mode;
3918
3919 if (FLOAT_WORDS_BIG_ENDIAN)
3920 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3921 else
3922 word = bitpos / BITS_PER_WORD;
3923 bitpos = bitpos % BITS_PER_WORD;
3924 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3925 }
3926
3927 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3928
3929 if (target == 0
3930 || target == op0
3931 || target == op1
3932 || reg_overlap_mentioned_p (target, op0)
3933 || reg_overlap_mentioned_p (target, op1)
3934 || (nwords > 1 && !valid_multiword_target_p (target)))
3935 target = gen_reg_rtx (mode);
3936
3937 if (nwords > 1)
3938 {
3939 start_sequence ();
3940
3941 for (i = 0; i < nwords; ++i)
3942 {
3943 rtx targ_piece = operand_subword (target, i, 1, mode);
3944 rtx op0_piece = operand_subword_force (op0, i, mode);
3945
3946 if (i == word)
3947 {
3948 if (!op0_is_abs)
3949 op0_piece
3950 = expand_binop (imode, and_optab, op0_piece,
3951 immed_wide_int_const (~mask, imode),
3952 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3953 op1 = expand_binop (imode, and_optab,
3954 operand_subword_force (op1, i, mode),
3955 immed_wide_int_const (mask, imode),
3956 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3957
3958 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3959 targ_piece, 1, OPTAB_LIB_WIDEN);
3960 if (temp != targ_piece)
3961 emit_move_insn (targ_piece, temp);
3962 }
3963 else
3964 emit_move_insn (targ_piece, op0_piece);
3965 }
3966
3967 insns = get_insns ();
3968 end_sequence ();
3969
3970 emit_insn (insns);
3971 }
3972 else
3973 {
3974 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3975 immed_wide_int_const (mask, imode),
3976 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3977
3978 op0 = gen_lowpart (imode, op0);
3979 if (!op0_is_abs)
3980 op0 = expand_binop (imode, and_optab, op0,
3981 immed_wide_int_const (~mask, imode),
3982 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3983
3984 temp = expand_binop (imode, ior_optab, op0, op1,
3985 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3986 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3987 }
3988
3989 return target;
3990 }
3991
3992 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3993 scalar floating point mode. Return NULL if we do not know how to
3994 expand the operation inline. */
3995
3996 rtx
expand_copysign(rtx op0,rtx op1,rtx target)3997 expand_copysign (rtx op0, rtx op1, rtx target)
3998 {
3999 scalar_float_mode mode;
4000 const struct real_format *fmt;
4001 bool op0_is_abs;
4002 rtx temp;
4003
4004 mode = as_a <scalar_float_mode> (GET_MODE (op0));
4005 gcc_assert (GET_MODE (op1) == mode);
4006
4007 /* First try to do it with a special instruction. */
4008 temp = expand_binop (mode, copysign_optab, op0, op1,
4009 target, 0, OPTAB_DIRECT);
4010 if (temp)
4011 return temp;
4012
4013 fmt = REAL_MODE_FORMAT (mode);
4014 if (fmt == NULL || !fmt->has_signed_zero)
4015 return NULL_RTX;
4016
4017 op0_is_abs = false;
4018 if (CONST_DOUBLE_AS_FLOAT_P (op0))
4019 {
4020 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4021 op0 = simplify_unary_operation (ABS, mode, op0, mode);
4022 op0_is_abs = true;
4023 }
4024
4025 if (fmt->signbit_ro >= 0
4026 && (CONST_DOUBLE_AS_FLOAT_P (op0)
4027 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
4028 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
4029 {
4030 temp = expand_copysign_absneg (mode, op0, op1, target,
4031 fmt->signbit_ro, op0_is_abs);
4032 if (temp)
4033 return temp;
4034 }
4035
4036 if (fmt->signbit_rw < 0)
4037 return NULL_RTX;
4038 return expand_copysign_bit (mode, op0, op1, target,
4039 fmt->signbit_rw, op0_is_abs);
4040 }
4041
4042 /* Generate an instruction whose insn-code is INSN_CODE,
4043 with two operands: an output TARGET and an input OP0.
4044 TARGET *must* be nonzero, and the output is always stored there.
4045 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4046 the value that is stored into TARGET.
4047
4048 Return false if expansion failed. */
4049
4050 bool
maybe_emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)4051 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4052 enum rtx_code code)
4053 {
4054 class expand_operand ops[2];
4055 rtx_insn *pat;
4056
4057 create_output_operand (&ops[0], target, GET_MODE (target));
4058 create_input_operand (&ops[1], op0, GET_MODE (op0));
4059 pat = maybe_gen_insn (icode, 2, ops);
4060 if (!pat)
4061 return false;
4062
4063 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
4064 && code != UNKNOWN)
4065 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
4066 GET_MODE (op0));
4067
4068 emit_insn (pat);
4069
4070 if (ops[0].value != target)
4071 emit_move_insn (target, ops[0].value);
4072 return true;
4073 }
4074 /* Generate an instruction whose insn-code is INSN_CODE,
4075 with two operands: an output TARGET and an input OP0.
4076 TARGET *must* be nonzero, and the output is always stored there.
4077 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4078 the value that is stored into TARGET. */
4079
4080 void
emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)4081 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4082 {
4083 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4084 gcc_assert (ok);
4085 }
4086
4087 struct no_conflict_data
4088 {
4089 rtx target;
4090 rtx_insn *first, *insn;
4091 bool must_stay;
4092 };
4093
4094 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4095 the currently examined clobber / store has to stay in the list of
4096 insns that constitute the actual libcall block. */
4097 static void
no_conflict_move_test(rtx dest,const_rtx set,void * p0)4098 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4099 {
4100 struct no_conflict_data *p= (struct no_conflict_data *) p0;
4101
4102 /* If this inns directly contributes to setting the target, it must stay. */
4103 if (reg_overlap_mentioned_p (p->target, dest))
4104 p->must_stay = true;
4105 /* If we haven't committed to keeping any other insns in the list yet,
4106 there is nothing more to check. */
4107 else if (p->insn == p->first)
4108 return;
4109 /* If this insn sets / clobbers a register that feeds one of the insns
4110 already in the list, this insn has to stay too. */
4111 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
4112 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4113 || reg_used_between_p (dest, p->first, p->insn)
4114 /* Likewise if this insn depends on a register set by a previous
4115 insn in the list, or if it sets a result (presumably a hard
4116 register) that is set or clobbered by a previous insn.
4117 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4118 SET_DEST perform the former check on the address, and the latter
4119 check on the MEM. */
4120 || (GET_CODE (set) == SET
4121 && (modified_in_p (SET_SRC (set), p->first)
4122 || modified_in_p (SET_DEST (set), p->first)
4123 || modified_between_p (SET_SRC (set), p->first, p->insn)
4124 || modified_between_p (SET_DEST (set), p->first, p->insn))))
4125 p->must_stay = true;
4126 }
4127
4128
4129 /* Emit code to make a call to a constant function or a library call.
4130
4131 INSNS is a list containing all insns emitted in the call.
4132 These insns leave the result in RESULT. Our block is to copy RESULT
4133 to TARGET, which is logically equivalent to EQUIV.
4134
4135 We first emit any insns that set a pseudo on the assumption that these are
4136 loading constants into registers; doing so allows them to be safely cse'ed
4137 between blocks. Then we emit all the other insns in the block, followed by
4138 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4139 note with an operand of EQUIV. */
4140
4141 static void
emit_libcall_block_1(rtx_insn * insns,rtx target,rtx result,rtx equiv,bool equiv_may_trap)4142 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4143 bool equiv_may_trap)
4144 {
4145 rtx final_dest = target;
4146 rtx_insn *next, *last, *insn;
4147
4148 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4149 into a MEM later. Protect the libcall block from this change. */
4150 if (! REG_P (target) || REG_USERVAR_P (target))
4151 target = gen_reg_rtx (GET_MODE (target));
4152
4153 /* If we're using non-call exceptions, a libcall corresponding to an
4154 operation that may trap may also trap. */
4155 /* ??? See the comment in front of make_reg_eh_region_note. */
4156 if (cfun->can_throw_non_call_exceptions
4157 && (equiv_may_trap || may_trap_p (equiv)))
4158 {
4159 for (insn = insns; insn; insn = NEXT_INSN (insn))
4160 if (CALL_P (insn))
4161 {
4162 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4163 if (note)
4164 {
4165 int lp_nr = INTVAL (XEXP (note, 0));
4166 if (lp_nr == 0 || lp_nr == INT_MIN)
4167 remove_note (insn, note);
4168 }
4169 }
4170 }
4171 else
4172 {
4173 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4174 reg note to indicate that this call cannot throw or execute a nonlocal
4175 goto (unless there is already a REG_EH_REGION note, in which case
4176 we update it). */
4177 for (insn = insns; insn; insn = NEXT_INSN (insn))
4178 if (CALL_P (insn))
4179 make_reg_eh_region_note_nothrow_nononlocal (insn);
4180 }
4181
4182 /* First emit all insns that set pseudos. Remove them from the list as
4183 we go. Avoid insns that set pseudos which were referenced in previous
4184 insns. These can be generated by move_by_pieces, for example,
4185 to update an address. Similarly, avoid insns that reference things
4186 set in previous insns. */
4187
4188 for (insn = insns; insn; insn = next)
4189 {
4190 rtx set = single_set (insn);
4191
4192 next = NEXT_INSN (insn);
4193
4194 if (set != 0 && REG_P (SET_DEST (set))
4195 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4196 {
4197 struct no_conflict_data data;
4198
4199 data.target = const0_rtx;
4200 data.first = insns;
4201 data.insn = insn;
4202 data.must_stay = 0;
4203 note_stores (insn, no_conflict_move_test, &data);
4204 if (! data.must_stay)
4205 {
4206 if (PREV_INSN (insn))
4207 SET_NEXT_INSN (PREV_INSN (insn)) = next;
4208 else
4209 insns = next;
4210
4211 if (next)
4212 SET_PREV_INSN (next) = PREV_INSN (insn);
4213
4214 add_insn (insn);
4215 }
4216 }
4217
4218 /* Some ports use a loop to copy large arguments onto the stack.
4219 Don't move anything outside such a loop. */
4220 if (LABEL_P (insn))
4221 break;
4222 }
4223
4224 /* Write the remaining insns followed by the final copy. */
4225 for (insn = insns; insn; insn = next)
4226 {
4227 next = NEXT_INSN (insn);
4228
4229 add_insn (insn);
4230 }
4231
4232 last = emit_move_insn (target, result);
4233 if (equiv)
4234 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4235
4236 if (final_dest != target)
4237 emit_move_insn (final_dest, target);
4238 }
4239
4240 void
emit_libcall_block(rtx_insn * insns,rtx target,rtx result,rtx equiv)4241 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4242 {
4243 emit_libcall_block_1 (insns, target, result, equiv, false);
4244 }
4245
4246 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
4247 PURPOSE describes how this comparison will be used. CODE is the rtx
4248 comparison code we will be using.
4249
4250 ??? Actually, CODE is slightly weaker than that. A target is still
4251 required to implement all of the normal bcc operations, but not
4252 required to implement all (or any) of the unordered bcc operations. */
4253
4254 int
can_compare_p(enum rtx_code code,machine_mode mode,enum can_compare_purpose purpose)4255 can_compare_p (enum rtx_code code, machine_mode mode,
4256 enum can_compare_purpose purpose)
4257 {
4258 rtx test;
4259 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4260 do
4261 {
4262 enum insn_code icode;
4263
4264 if (purpose == ccp_jump
4265 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4266 && insn_operand_matches (icode, 0, test))
4267 return 1;
4268 if (purpose == ccp_store_flag
4269 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4270 && insn_operand_matches (icode, 1, test))
4271 return 1;
4272 if (purpose == ccp_cmov
4273 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
4274 return 1;
4275
4276 mode = GET_MODE_WIDER_MODE (mode).else_void ();
4277 PUT_MODE (test, mode);
4278 }
4279 while (mode != VOIDmode);
4280
4281 return 0;
4282 }
4283
4284 /* Return whether RTL code CODE corresponds to an unsigned optab. */
4285
4286 static bool
unsigned_optab_p(enum rtx_code code)4287 unsigned_optab_p (enum rtx_code code)
4288 {
4289 return code == LTU || code == LEU || code == GTU || code == GEU;
4290 }
4291
4292 /* Return whether the backend-emitted comparison for code CODE, comparing
4293 operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4294 operand OPNO of pattern ICODE. */
4295
4296 static bool
insn_predicate_matches_p(enum insn_code icode,unsigned int opno,enum rtx_code code,machine_mode mask_mode,machine_mode value_mode)4297 insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4298 enum rtx_code code, machine_mode mask_mode,
4299 machine_mode value_mode)
4300 {
4301 rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4302 rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4303 rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4304 return insn_operand_matches (icode, opno, test);
4305 }
4306
4307 /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4308 for code CODE, comparing operands of mode VALUE_MODE and producing a result
4309 with MASK_MODE. */
4310
4311 bool
can_vec_cmp_compare_p(enum rtx_code code,machine_mode value_mode,machine_mode mask_mode)4312 can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4313 machine_mode mask_mode)
4314 {
4315 enum insn_code icode
4316 = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
4317 if (icode == CODE_FOR_nothing)
4318 return false;
4319
4320 return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
4321 }
4322
4323 /* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4324 code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4325 with VALUE_MODE. */
4326
4327 bool
can_vcond_compare_p(enum rtx_code code,machine_mode value_mode,machine_mode cmp_op_mode)4328 can_vcond_compare_p (enum rtx_code code, machine_mode value_mode,
4329 machine_mode cmp_op_mode)
4330 {
4331 enum insn_code icode
4332 = get_vcond_icode (value_mode, cmp_op_mode, unsigned_optab_p (code));
4333 if (icode == CODE_FOR_nothing)
4334 return false;
4335
4336 return insn_predicate_matches_p (icode, 3, code, value_mode, cmp_op_mode);
4337 }
4338
4339 /* Return whether the backend can emit vector set instructions for inserting
4340 element into vector at variable index position. */
4341
4342 bool
can_vec_set_var_idx_p(machine_mode vec_mode)4343 can_vec_set_var_idx_p (machine_mode vec_mode)
4344 {
4345 if (!VECTOR_MODE_P (vec_mode))
4346 return false;
4347
4348 machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4349 rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4350 rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4351 rtx reg3 = alloca_raw_REG (VOIDmode, LAST_VIRTUAL_REGISTER + 3);
4352
4353 enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
4354
4355 return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4356 && insn_operand_matches (icode, 1, reg2)
4357 && insn_operand_matches (icode, 2, reg3);
4358 }
4359
4360 /* This function is called when we are going to emit a compare instruction that
4361 compares the values found in X and Y, using the rtl operator COMPARISON.
4362
4363 If they have mode BLKmode, then SIZE specifies the size of both operands.
4364
4365 UNSIGNEDP nonzero says that the operands are unsigned;
4366 this matters if they need to be widened (as given by METHODS).
4367
4368 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4369 if we failed to produce one.
4370
4371 *PMODE is the mode of the inputs (in case they are const_int).
4372
4373 This function performs all the setup necessary so that the caller only has
4374 to emit a single comparison insn. This setup can involve doing a BLKmode
4375 comparison or emitting a library call to perform the comparison if no insn
4376 is available to handle it.
4377 The values which are passed in through pointers can be modified; the caller
4378 should perform the comparison on the modified values. Constant
4379 comparisons must have already been folded. */
4380
4381 static void
prepare_cmp_insn(rtx x,rtx y,enum rtx_code comparison,rtx size,int unsignedp,enum optab_methods methods,rtx * ptest,machine_mode * pmode)4382 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
4383 int unsignedp, enum optab_methods methods,
4384 rtx *ptest, machine_mode *pmode)
4385 {
4386 machine_mode mode = *pmode;
4387 rtx libfunc, test;
4388 machine_mode cmp_mode;
4389 enum mode_class mclass;
4390
4391 /* The other methods are not needed. */
4392 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4393 || methods == OPTAB_LIB_WIDEN);
4394
4395 if (CONST_SCALAR_INT_P (y))
4396 canonicalize_comparison (mode, &comparison, &y);
4397
4398 /* If we are optimizing, force expensive constants into a register. */
4399 if (CONSTANT_P (x) && optimize
4400 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4401 > COSTS_N_INSNS (1))
4402 && can_create_pseudo_p ())
4403 x = force_reg (mode, x);
4404
4405 if (CONSTANT_P (y) && optimize
4406 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4407 > COSTS_N_INSNS (1))
4408 && can_create_pseudo_p ())
4409 y = force_reg (mode, y);
4410
4411 /* Don't let both operands fail to indicate the mode. */
4412 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4413 x = force_reg (mode, x);
4414 if (mode == VOIDmode)
4415 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4416
4417 /* Handle all BLKmode compares. */
4418
4419 if (mode == BLKmode)
4420 {
4421 machine_mode result_mode;
4422 enum insn_code cmp_code;
4423 rtx result;
4424 rtx opalign
4425 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4426
4427 gcc_assert (size);
4428
4429 /* Try to use a memory block compare insn - either cmpstr
4430 or cmpmem will do. */
4431 opt_scalar_int_mode cmp_mode_iter;
4432 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4433 {
4434 scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4435 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4436 if (cmp_code == CODE_FOR_nothing)
4437 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4438 if (cmp_code == CODE_FOR_nothing)
4439 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4440 if (cmp_code == CODE_FOR_nothing)
4441 continue;
4442
4443 /* Must make sure the size fits the insn's mode. */
4444 if (CONST_INT_P (size)
4445 ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4446 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
4447 > GET_MODE_BITSIZE (cmp_mode)))
4448 continue;
4449
4450 result_mode = insn_data[cmp_code].operand[0].mode;
4451 result = gen_reg_rtx (result_mode);
4452 size = convert_to_mode (cmp_mode, size, 1);
4453 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4454
4455 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4456 *pmode = result_mode;
4457 return;
4458 }
4459
4460 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4461 goto fail;
4462
4463 /* Otherwise call a library function. */
4464 result = emit_block_comp_via_libcall (x, y, size);
4465
4466 x = result;
4467 y = const0_rtx;
4468 mode = TYPE_MODE (integer_type_node);
4469 methods = OPTAB_LIB_WIDEN;
4470 unsignedp = false;
4471 }
4472
4473 /* Don't allow operands to the compare to trap, as that can put the
4474 compare and branch in different basic blocks. */
4475 if (cfun->can_throw_non_call_exceptions)
4476 {
4477 if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4478 goto fail;
4479 if (may_trap_p (x))
4480 x = copy_to_reg (x);
4481 if (may_trap_p (y))
4482 y = copy_to_reg (y);
4483 }
4484
4485 if (GET_MODE_CLASS (mode) == MODE_CC)
4486 {
4487 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4488 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4489 gcc_assert (icode != CODE_FOR_nothing
4490 && insn_operand_matches (icode, 0, test));
4491 *ptest = test;
4492 return;
4493 }
4494
4495 mclass = GET_MODE_CLASS (mode);
4496 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4497 FOR_EACH_MODE_FROM (cmp_mode, mode)
4498 {
4499 enum insn_code icode;
4500 icode = optab_handler (cbranch_optab, cmp_mode);
4501 if (icode != CODE_FOR_nothing
4502 && insn_operand_matches (icode, 0, test))
4503 {
4504 rtx_insn *last = get_last_insn ();
4505 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4506 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4507 if (op0 && op1
4508 && insn_operand_matches (icode, 1, op0)
4509 && insn_operand_matches (icode, 2, op1))
4510 {
4511 XEXP (test, 0) = op0;
4512 XEXP (test, 1) = op1;
4513 *ptest = test;
4514 *pmode = cmp_mode;
4515 return;
4516 }
4517 delete_insns_since (last);
4518 }
4519
4520 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4521 break;
4522 }
4523
4524 if (methods != OPTAB_LIB_WIDEN)
4525 goto fail;
4526
4527 if (SCALAR_FLOAT_MODE_P (mode))
4528 {
4529 /* Small trick if UNORDERED isn't implemented by the hardware. */
4530 if (comparison == UNORDERED && rtx_equal_p (x, y))
4531 {
4532 prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4533 ptest, pmode);
4534 if (*ptest)
4535 return;
4536 }
4537
4538 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4539 }
4540 else
4541 {
4542 rtx result;
4543 machine_mode ret_mode;
4544
4545 /* Handle a libcall just for the mode we are using. */
4546 libfunc = optab_libfunc (cmp_optab, mode);
4547 gcc_assert (libfunc);
4548
4549 /* If we want unsigned, and this mode has a distinct unsigned
4550 comparison routine, use that. */
4551 if (unsignedp)
4552 {
4553 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4554 if (ulibfunc)
4555 libfunc = ulibfunc;
4556 }
4557
4558 ret_mode = targetm.libgcc_cmp_return_mode ();
4559 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4560 ret_mode, x, mode, y, mode);
4561
4562 /* There are two kinds of comparison routines. Biased routines
4563 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4564 of gcc expect that the comparison operation is equivalent
4565 to the modified comparison. For signed comparisons compare the
4566 result against 1 in the biased case, and zero in the unbiased
4567 case. For unsigned comparisons always compare against 1 after
4568 biasing the unbiased result by adding 1. This gives us a way to
4569 represent LTU.
4570 The comparisons in the fixed-point helper library are always
4571 biased. */
4572 x = result;
4573 y = const1_rtx;
4574
4575 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4576 {
4577 if (unsignedp)
4578 x = plus_constant (ret_mode, result, 1);
4579 else
4580 y = const0_rtx;
4581 }
4582
4583 *pmode = ret_mode;
4584 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4585 ptest, pmode);
4586 }
4587
4588 return;
4589
4590 fail:
4591 *ptest = NULL_RTX;
4592 }
4593
4594 /* Before emitting an insn with code ICODE, make sure that X, which is going
4595 to be used for operand OPNUM of the insn, is converted from mode MODE to
4596 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4597 that it is accepted by the operand predicate. Return the new value. */
4598
4599 rtx
prepare_operand(enum insn_code icode,rtx x,int opnum,machine_mode mode,machine_mode wider_mode,int unsignedp)4600 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4601 machine_mode wider_mode, int unsignedp)
4602 {
4603 if (mode != wider_mode)
4604 x = convert_modes (wider_mode, mode, x, unsignedp);
4605
4606 if (!insn_operand_matches (icode, opnum, x))
4607 {
4608 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4609 if (reload_completed)
4610 return NULL_RTX;
4611 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4612 return NULL_RTX;
4613 x = copy_to_mode_reg (op_mode, x);
4614 }
4615
4616 return x;
4617 }
4618
4619 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4620 we can do the branch. */
4621
4622 static void
emit_cmp_and_jump_insn_1(rtx test,machine_mode mode,rtx label,profile_probability prob)4623 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4624 profile_probability prob)
4625 {
4626 machine_mode optab_mode;
4627 enum mode_class mclass;
4628 enum insn_code icode;
4629 rtx_insn *insn;
4630
4631 mclass = GET_MODE_CLASS (mode);
4632 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4633 icode = optab_handler (cbranch_optab, optab_mode);
4634
4635 gcc_assert (icode != CODE_FOR_nothing);
4636 gcc_assert (insn_operand_matches (icode, 0, test));
4637 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4638 XEXP (test, 1), label));
4639 if (prob.initialized_p ()
4640 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4641 && insn
4642 && JUMP_P (insn)
4643 && any_condjump_p (insn)
4644 && !find_reg_note (insn, REG_BR_PROB, 0))
4645 add_reg_br_prob_note (insn, prob);
4646 }
4647
4648 /* Generate code to compare X with Y so that the condition codes are
4649 set and to jump to LABEL if the condition is true. If X is a
4650 constant and Y is not a constant, then the comparison is swapped to
4651 ensure that the comparison RTL has the canonical form.
4652
4653 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4654 need to be widened. UNSIGNEDP is also used to select the proper
4655 branch condition code.
4656
4657 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4658
4659 MODE is the mode of the inputs (in case they are const_int).
4660
4661 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4662 It will be potentially converted into an unsigned variant based on
4663 UNSIGNEDP to select a proper jump instruction.
4664
4665 PROB is the probability of jumping to LABEL. */
4666
4667 void
emit_cmp_and_jump_insns(rtx x,rtx y,enum rtx_code comparison,rtx size,machine_mode mode,int unsignedp,rtx label,profile_probability prob)4668 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4669 machine_mode mode, int unsignedp, rtx label,
4670 profile_probability prob)
4671 {
4672 rtx op0 = x, op1 = y;
4673 rtx test;
4674
4675 /* Swap operands and condition to ensure canonical RTL. */
4676 if (swap_commutative_operands_p (x, y)
4677 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4678 {
4679 op0 = y, op1 = x;
4680 comparison = swap_condition (comparison);
4681 }
4682
4683 /* If OP0 is still a constant, then both X and Y must be constants
4684 or the opposite comparison is not supported. Force X into a register
4685 to create canonical RTL. */
4686 if (CONSTANT_P (op0))
4687 op0 = force_reg (mode, op0);
4688
4689 if (unsignedp)
4690 comparison = unsigned_condition (comparison);
4691
4692 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4693 &test, &mode);
4694 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4695 }
4696
4697
4698 /* Emit a library call comparison between floating point X and Y.
4699 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4700
4701 static void
prepare_float_lib_cmp(rtx x,rtx y,enum rtx_code comparison,rtx * ptest,machine_mode * pmode)4702 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4703 rtx *ptest, machine_mode *pmode)
4704 {
4705 enum rtx_code swapped = swap_condition (comparison);
4706 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4707 machine_mode orig_mode = GET_MODE (x);
4708 machine_mode mode;
4709 rtx true_rtx, false_rtx;
4710 rtx value, target, equiv;
4711 rtx_insn *insns;
4712 rtx libfunc = 0;
4713 bool reversed_p = false;
4714 scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4715
4716 FOR_EACH_MODE_FROM (mode, orig_mode)
4717 {
4718 if (code_to_optab (comparison)
4719 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4720 break;
4721
4722 if (code_to_optab (swapped)
4723 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4724 {
4725 std::swap (x, y);
4726 comparison = swapped;
4727 break;
4728 }
4729
4730 if (code_to_optab (reversed)
4731 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4732 {
4733 comparison = reversed;
4734 reversed_p = true;
4735 break;
4736 }
4737 }
4738
4739 gcc_assert (mode != VOIDmode);
4740
4741 if (mode != orig_mode)
4742 {
4743 x = convert_to_mode (mode, x, 0);
4744 y = convert_to_mode (mode, y, 0);
4745 }
4746
4747 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4748 the RTL. The allows the RTL optimizers to delete the libcall if the
4749 condition can be determined at compile-time. */
4750 if (comparison == UNORDERED
4751 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4752 {
4753 true_rtx = const_true_rtx;
4754 false_rtx = const0_rtx;
4755 }
4756 else
4757 {
4758 switch (comparison)
4759 {
4760 case EQ:
4761 true_rtx = const0_rtx;
4762 false_rtx = const_true_rtx;
4763 break;
4764
4765 case NE:
4766 true_rtx = const_true_rtx;
4767 false_rtx = const0_rtx;
4768 break;
4769
4770 case GT:
4771 true_rtx = const1_rtx;
4772 false_rtx = const0_rtx;
4773 break;
4774
4775 case GE:
4776 true_rtx = const0_rtx;
4777 false_rtx = constm1_rtx;
4778 break;
4779
4780 case LT:
4781 true_rtx = constm1_rtx;
4782 false_rtx = const0_rtx;
4783 break;
4784
4785 case LE:
4786 true_rtx = const0_rtx;
4787 false_rtx = const1_rtx;
4788 break;
4789
4790 default:
4791 gcc_unreachable ();
4792 }
4793 }
4794
4795 if (comparison == UNORDERED)
4796 {
4797 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4798 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4799 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4800 temp, const_true_rtx, equiv);
4801 }
4802 else
4803 {
4804 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4805 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4806 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4807 equiv, true_rtx, false_rtx);
4808 }
4809
4810 start_sequence ();
4811 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4812 cmp_mode, x, mode, y, mode);
4813 insns = get_insns ();
4814 end_sequence ();
4815
4816 target = gen_reg_rtx (cmp_mode);
4817 emit_libcall_block (insns, target, value, equiv);
4818
4819 if (comparison == UNORDERED
4820 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4821 || reversed_p)
4822 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4823 else
4824 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4825
4826 *pmode = cmp_mode;
4827 }
4828
4829 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4830
4831 void
emit_indirect_jump(rtx loc)4832 emit_indirect_jump (rtx loc)
4833 {
4834 if (!targetm.have_indirect_jump ())
4835 sorry ("indirect jumps are not available on this target");
4836 else
4837 {
4838 class expand_operand ops[1];
4839 create_address_operand (&ops[0], loc);
4840 expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
4841 emit_barrier ();
4842 }
4843 }
4844
4845
4846 /* Emit a conditional move instruction if the machine supports one for that
4847 condition and machine mode.
4848
4849 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4850 the mode to use should they be constants. If it is VOIDmode, they cannot
4851 both be constants.
4852
4853 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4854 should be stored there. MODE is the mode to use should they be constants.
4855 If it is VOIDmode, they cannot both be constants.
4856
4857 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4858 is not supported. */
4859
4860 rtx
emit_conditional_move(rtx target,struct rtx_comparison comp,rtx op2,rtx op3,machine_mode mode,int unsignedp)4861 emit_conditional_move (rtx target, struct rtx_comparison comp,
4862 rtx op2, rtx op3,
4863 machine_mode mode, int unsignedp)
4864 {
4865 rtx comparison;
4866 rtx_insn *last;
4867 enum insn_code icode;
4868 enum rtx_code reversed;
4869
4870 /* If the two source operands are identical, that's just a move. */
4871
4872 if (rtx_equal_p (op2, op3))
4873 {
4874 if (!target)
4875 target = gen_reg_rtx (mode);
4876
4877 emit_move_insn (target, op3);
4878 return target;
4879 }
4880
4881 /* If one operand is constant, make it the second one. Only do this
4882 if the other operand is not constant as well. */
4883
4884 if (swap_commutative_operands_p (comp.op0, comp.op1))
4885 {
4886 std::swap (comp.op0, comp.op1);
4887 comp.code = swap_condition (comp.code);
4888 }
4889
4890 /* get_condition will prefer to generate LT and GT even if the old
4891 comparison was against zero, so undo that canonicalization here since
4892 comparisons against zero are cheaper. */
4893
4894 if (comp.code == LT && comp.op1 == const1_rtx)
4895 comp.code = LE, comp.op1 = const0_rtx;
4896 else if (comp.code == GT && comp.op1 == constm1_rtx)
4897 comp.code = GE, comp.op1 = const0_rtx;
4898
4899 if (comp.mode == VOIDmode)
4900 comp.mode = GET_MODE (comp.op0);
4901
4902 enum rtx_code orig_code = comp.code;
4903 bool swapped = false;
4904 if (swap_commutative_operands_p (op2, op3)
4905 && ((reversed =
4906 reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
4907 != UNKNOWN))
4908 {
4909 std::swap (op2, op3);
4910 comp.code = reversed;
4911 swapped = true;
4912 }
4913
4914 if (mode == VOIDmode)
4915 mode = GET_MODE (op2);
4916
4917 icode = direct_optab_handler (movcc_optab, mode);
4918
4919 if (icode == CODE_FOR_nothing)
4920 return NULL_RTX;
4921
4922 if (!target)
4923 target = gen_reg_rtx (mode);
4924
4925 for (int pass = 0; ; pass++)
4926 {
4927 comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
4928 comparison =
4929 simplify_gen_relational (comp.code, VOIDmode,
4930 comp.mode, comp.op0, comp.op1);
4931
4932 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4933 punt and let the caller figure out how best to deal with this
4934 situation. */
4935 if (COMPARISON_P (comparison))
4936 {
4937 saved_pending_stack_adjust save;
4938 save_pending_stack_adjust (&save);
4939 last = get_last_insn ();
4940 do_pending_stack_adjust ();
4941 machine_mode cmpmode = comp.mode;
4942 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4943 GET_CODE (comparison), NULL_RTX, unsignedp,
4944 OPTAB_WIDEN, &comparison, &cmpmode);
4945 if (comparison)
4946 {
4947 rtx res = emit_conditional_move_1 (target, comparison,
4948 op2, op3, mode);
4949 if (res != NULL_RTX)
4950 return res;
4951 }
4952 delete_insns_since (last);
4953 restore_pending_stack_adjust (&save);
4954 }
4955
4956 if (pass == 1)
4957 return NULL_RTX;
4958
4959 /* If the preferred op2/op3 order is not usable, retry with other
4960 operand order, perhaps it will expand successfully. */
4961 if (swapped)
4962 comp.code = orig_code;
4963 else if ((reversed =
4964 reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
4965 NULL))
4966 != UNKNOWN)
4967 comp.code = reversed;
4968 else
4969 return NULL_RTX;
4970 std::swap (op2, op3);
4971 }
4972 }
4973
4974 /* Helper function that, in addition to COMPARISON, also tries
4975 the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
4976 to when we pass the specific constituents of a comparison, no
4977 additional insns are emitted for it. It might still be necessary
4978 to emit more than one insn for the final conditional move, though. */
4979
4980 rtx
emit_conditional_move(rtx target,rtx comparison,rtx rev_comparison,rtx op2,rtx op3,machine_mode mode)4981 emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
4982 rtx op2, rtx op3, machine_mode mode)
4983 {
4984 rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
4985
4986 if (res != NULL_RTX)
4987 return res;
4988
4989 return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
4990 }
4991
4992 /* Helper for emitting a conditional move. */
4993
4994 static rtx
emit_conditional_move_1(rtx target,rtx comparison,rtx op2,rtx op3,machine_mode mode)4995 emit_conditional_move_1 (rtx target, rtx comparison,
4996 rtx op2, rtx op3, machine_mode mode)
4997 {
4998 enum insn_code icode;
4999
5000 if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5001 return NULL_RTX;
5002
5003 /* If the two source operands are identical, that's just a move.
5004 As the comparison comes in non-canonicalized, we must make
5005 sure not to discard any possible side effects. If there are
5006 side effects, just let the target handle it. */
5007 if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5008 {
5009 if (!target)
5010 target = gen_reg_rtx (mode);
5011
5012 emit_move_insn (target, op3);
5013 return target;
5014 }
5015
5016 if (mode == VOIDmode)
5017 mode = GET_MODE (op2);
5018
5019 icode = direct_optab_handler (movcc_optab, mode);
5020
5021 if (icode == CODE_FOR_nothing)
5022 return NULL_RTX;
5023
5024 if (!target)
5025 target = gen_reg_rtx (mode);
5026
5027 class expand_operand ops[4];
5028
5029 create_output_operand (&ops[0], target, mode);
5030 create_fixed_operand (&ops[1], comparison);
5031 create_input_operand (&ops[2], op2, mode);
5032 create_input_operand (&ops[3], op3, mode);
5033
5034 if (maybe_expand_insn (icode, 4, ops))
5035 {
5036 if (ops[0].value != target)
5037 convert_move (target, ops[0].value, false);
5038 return target;
5039 }
5040
5041 return NULL_RTX;
5042 }
5043
5044
5045 /* Emit a conditional negate or bitwise complement using the
5046 negcc or notcc optabs if available. Return NULL_RTX if such operations
5047 are not available. Otherwise return the RTX holding the result.
5048 TARGET is the desired destination of the result. COMP is the comparison
5049 on which to negate. If COND is true move into TARGET the negation
5050 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5051 CODE is either NEG or NOT. MODE is the machine mode in which the
5052 operation is performed. */
5053
5054 rtx
emit_conditional_neg_or_complement(rtx target,rtx_code code,machine_mode mode,rtx cond,rtx op1,rtx op2)5055 emit_conditional_neg_or_complement (rtx target, rtx_code code,
5056 machine_mode mode, rtx cond, rtx op1,
5057 rtx op2)
5058 {
5059 optab op = unknown_optab;
5060 if (code == NEG)
5061 op = negcc_optab;
5062 else if (code == NOT)
5063 op = notcc_optab;
5064 else
5065 gcc_unreachable ();
5066
5067 insn_code icode = direct_optab_handler (op, mode);
5068
5069 if (icode == CODE_FOR_nothing)
5070 return NULL_RTX;
5071
5072 if (!target)
5073 target = gen_reg_rtx (mode);
5074
5075 rtx_insn *last = get_last_insn ();
5076 class expand_operand ops[4];
5077
5078 create_output_operand (&ops[0], target, mode);
5079 create_fixed_operand (&ops[1], cond);
5080 create_input_operand (&ops[2], op1, mode);
5081 create_input_operand (&ops[3], op2, mode);
5082
5083 if (maybe_expand_insn (icode, 4, ops))
5084 {
5085 if (ops[0].value != target)
5086 convert_move (target, ops[0].value, false);
5087
5088 return target;
5089 }
5090 delete_insns_since (last);
5091 return NULL_RTX;
5092 }
5093
5094 /* Emit a conditional addition instruction if the machine supports one for that
5095 condition and machine mode.
5096
5097 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5098 the mode to use should they be constants. If it is VOIDmode, they cannot
5099 both be constants.
5100
5101 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5102 should be stored there. MODE is the mode to use should they be constants.
5103 If it is VOIDmode, they cannot both be constants.
5104
5105 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5106 is not supported. */
5107
5108 rtx
emit_conditional_add(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)5109 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5110 machine_mode cmode, rtx op2, rtx op3,
5111 machine_mode mode, int unsignedp)
5112 {
5113 rtx comparison;
5114 rtx_insn *last;
5115 enum insn_code icode;
5116
5117 /* If one operand is constant, make it the second one. Only do this
5118 if the other operand is not constant as well. */
5119
5120 if (swap_commutative_operands_p (op0, op1))
5121 {
5122 std::swap (op0, op1);
5123 code = swap_condition (code);
5124 }
5125
5126 /* get_condition will prefer to generate LT and GT even if the old
5127 comparison was against zero, so undo that canonicalization here since
5128 comparisons against zero are cheaper. */
5129 if (code == LT && op1 == const1_rtx)
5130 code = LE, op1 = const0_rtx;
5131 else if (code == GT && op1 == constm1_rtx)
5132 code = GE, op1 = const0_rtx;
5133
5134 if (cmode == VOIDmode)
5135 cmode = GET_MODE (op0);
5136
5137 if (mode == VOIDmode)
5138 mode = GET_MODE (op2);
5139
5140 icode = optab_handler (addcc_optab, mode);
5141
5142 if (icode == CODE_FOR_nothing)
5143 return 0;
5144
5145 if (!target)
5146 target = gen_reg_rtx (mode);
5147
5148 code = unsignedp ? unsigned_condition (code) : code;
5149 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
5150
5151 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5152 return NULL and let the caller figure out how best to deal with this
5153 situation. */
5154 if (!COMPARISON_P (comparison))
5155 return NULL_RTX;
5156
5157 do_pending_stack_adjust ();
5158 last = get_last_insn ();
5159 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
5160 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
5161 &comparison, &cmode);
5162 if (comparison)
5163 {
5164 class expand_operand ops[4];
5165
5166 create_output_operand (&ops[0], target, mode);
5167 create_fixed_operand (&ops[1], comparison);
5168 create_input_operand (&ops[2], op2, mode);
5169 create_input_operand (&ops[3], op3, mode);
5170 if (maybe_expand_insn (icode, 4, ops))
5171 {
5172 if (ops[0].value != target)
5173 convert_move (target, ops[0].value, false);
5174 return target;
5175 }
5176 }
5177 delete_insns_since (last);
5178 return NULL_RTX;
5179 }
5180
5181 /* These functions attempt to generate an insn body, rather than
5182 emitting the insn, but if the gen function already emits them, we
5183 make no attempt to turn them back into naked patterns. */
5184
5185 /* Generate and return an insn body to add Y to X. */
5186
5187 rtx_insn *
gen_add2_insn(rtx x,rtx y)5188 gen_add2_insn (rtx x, rtx y)
5189 {
5190 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
5191
5192 gcc_assert (insn_operand_matches (icode, 0, x));
5193 gcc_assert (insn_operand_matches (icode, 1, x));
5194 gcc_assert (insn_operand_matches (icode, 2, y));
5195
5196 return GEN_FCN (icode) (x, x, y);
5197 }
5198
5199 /* Generate and return an insn body to add r1 and c,
5200 storing the result in r0. */
5201
5202 rtx_insn *
gen_add3_insn(rtx r0,rtx r1,rtx c)5203 gen_add3_insn (rtx r0, rtx r1, rtx c)
5204 {
5205 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
5206
5207 if (icode == CODE_FOR_nothing
5208 || !insn_operand_matches (icode, 0, r0)
5209 || !insn_operand_matches (icode, 1, r1)
5210 || !insn_operand_matches (icode, 2, c))
5211 return NULL;
5212
5213 return GEN_FCN (icode) (r0, r1, c);
5214 }
5215
5216 int
have_add2_insn(rtx x,rtx y)5217 have_add2_insn (rtx x, rtx y)
5218 {
5219 enum insn_code icode;
5220
5221 gcc_assert (GET_MODE (x) != VOIDmode);
5222
5223 icode = optab_handler (add_optab, GET_MODE (x));
5224
5225 if (icode == CODE_FOR_nothing)
5226 return 0;
5227
5228 if (!insn_operand_matches (icode, 0, x)
5229 || !insn_operand_matches (icode, 1, x)
5230 || !insn_operand_matches (icode, 2, y))
5231 return 0;
5232
5233 return 1;
5234 }
5235
5236 /* Generate and return an insn body to add Y to X. */
5237
5238 rtx_insn *
gen_addptr3_insn(rtx x,rtx y,rtx z)5239 gen_addptr3_insn (rtx x, rtx y, rtx z)
5240 {
5241 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
5242
5243 gcc_assert (insn_operand_matches (icode, 0, x));
5244 gcc_assert (insn_operand_matches (icode, 1, y));
5245 gcc_assert (insn_operand_matches (icode, 2, z));
5246
5247 return GEN_FCN (icode) (x, y, z);
5248 }
5249
5250 /* Return true if the target implements an addptr pattern and X, Y,
5251 and Z are valid for the pattern predicates. */
5252
5253 int
have_addptr3_insn(rtx x,rtx y,rtx z)5254 have_addptr3_insn (rtx x, rtx y, rtx z)
5255 {
5256 enum insn_code icode;
5257
5258 gcc_assert (GET_MODE (x) != VOIDmode);
5259
5260 icode = optab_handler (addptr3_optab, GET_MODE (x));
5261
5262 if (icode == CODE_FOR_nothing)
5263 return 0;
5264
5265 if (!insn_operand_matches (icode, 0, x)
5266 || !insn_operand_matches (icode, 1, y)
5267 || !insn_operand_matches (icode, 2, z))
5268 return 0;
5269
5270 return 1;
5271 }
5272
5273 /* Generate and return an insn body to subtract Y from X. */
5274
5275 rtx_insn *
gen_sub2_insn(rtx x,rtx y)5276 gen_sub2_insn (rtx x, rtx y)
5277 {
5278 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
5279
5280 gcc_assert (insn_operand_matches (icode, 0, x));
5281 gcc_assert (insn_operand_matches (icode, 1, x));
5282 gcc_assert (insn_operand_matches (icode, 2, y));
5283
5284 return GEN_FCN (icode) (x, x, y);
5285 }
5286
5287 /* Generate and return an insn body to subtract r1 and c,
5288 storing the result in r0. */
5289
5290 rtx_insn *
gen_sub3_insn(rtx r0,rtx r1,rtx c)5291 gen_sub3_insn (rtx r0, rtx r1, rtx c)
5292 {
5293 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
5294
5295 if (icode == CODE_FOR_nothing
5296 || !insn_operand_matches (icode, 0, r0)
5297 || !insn_operand_matches (icode, 1, r1)
5298 || !insn_operand_matches (icode, 2, c))
5299 return NULL;
5300
5301 return GEN_FCN (icode) (r0, r1, c);
5302 }
5303
5304 int
have_sub2_insn(rtx x,rtx y)5305 have_sub2_insn (rtx x, rtx y)
5306 {
5307 enum insn_code icode;
5308
5309 gcc_assert (GET_MODE (x) != VOIDmode);
5310
5311 icode = optab_handler (sub_optab, GET_MODE (x));
5312
5313 if (icode == CODE_FOR_nothing)
5314 return 0;
5315
5316 if (!insn_operand_matches (icode, 0, x)
5317 || !insn_operand_matches (icode, 1, x)
5318 || !insn_operand_matches (icode, 2, y))
5319 return 0;
5320
5321 return 1;
5322 }
5323
5324 /* Generate the body of an insn to extend Y (with mode MFROM)
5325 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5326
5327 rtx_insn *
gen_extend_insn(rtx x,rtx y,machine_mode mto,machine_mode mfrom,int unsignedp)5328 gen_extend_insn (rtx x, rtx y, machine_mode mto,
5329 machine_mode mfrom, int unsignedp)
5330 {
5331 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5332 return GEN_FCN (icode) (x, y);
5333 }
5334
5335 /* Generate code to convert FROM to floating point
5336 and store in TO. FROM must be fixed point and not VOIDmode.
5337 UNSIGNEDP nonzero means regard FROM as unsigned.
5338 Normally this is done by correcting the final value
5339 if it is negative. */
5340
5341 void
expand_float(rtx to,rtx from,int unsignedp)5342 expand_float (rtx to, rtx from, int unsignedp)
5343 {
5344 enum insn_code icode;
5345 rtx target = to;
5346 scalar_mode from_mode, to_mode;
5347 machine_mode fmode, imode;
5348 bool can_do_signed = false;
5349
5350 /* Crash now, because we won't be able to decide which mode to use. */
5351 gcc_assert (GET_MODE (from) != VOIDmode);
5352
5353 /* Look for an insn to do the conversion. Do it in the specified
5354 modes if possible; otherwise convert either input, output or both to
5355 wider mode. If the integer mode is wider than the mode of FROM,
5356 we can do the conversion signed even if the input is unsigned. */
5357
5358 FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5359 FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5360 {
5361 int doing_unsigned = unsignedp;
5362
5363 if (fmode != GET_MODE (to)
5364 && (significand_size (fmode)
5365 < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5366 continue;
5367
5368 icode = can_float_p (fmode, imode, unsignedp);
5369 if (icode == CODE_FOR_nothing && unsignedp)
5370 {
5371 enum insn_code scode = can_float_p (fmode, imode, 0);
5372 if (scode != CODE_FOR_nothing)
5373 can_do_signed = true;
5374 if (imode != GET_MODE (from))
5375 icode = scode, doing_unsigned = 0;
5376 }
5377
5378 if (icode != CODE_FOR_nothing)
5379 {
5380 if (imode != GET_MODE (from))
5381 from = convert_to_mode (imode, from, unsignedp);
5382
5383 if (fmode != GET_MODE (to))
5384 target = gen_reg_rtx (fmode);
5385
5386 emit_unop_insn (icode, target, from,
5387 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5388
5389 if (target != to)
5390 convert_move (to, target, 0);
5391 return;
5392 }
5393 }
5394
5395 /* Unsigned integer, and no way to convert directly. Convert as signed,
5396 then unconditionally adjust the result. */
5397 if (unsignedp
5398 && can_do_signed
5399 && is_a <scalar_mode> (GET_MODE (to), &to_mode)
5400 && is_a <scalar_mode> (GET_MODE (from), &from_mode))
5401 {
5402 opt_scalar_mode fmode_iter;
5403 rtx_code_label *label = gen_label_rtx ();
5404 rtx temp;
5405 REAL_VALUE_TYPE offset;
5406
5407 /* Look for a usable floating mode FMODE wider than the source and at
5408 least as wide as the target. Using FMODE will avoid rounding woes
5409 with unsigned values greater than the signed maximum value. */
5410
5411 FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5412 {
5413 scalar_mode fmode = fmode_iter.require ();
5414 if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
5415 && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5416 break;
5417 }
5418
5419 if (!fmode_iter.exists (&fmode))
5420 {
5421 /* There is no such mode. Pretend the target is wide enough. */
5422 fmode = to_mode;
5423
5424 /* Avoid double-rounding when TO is narrower than FROM. */
5425 if ((significand_size (fmode) + 1)
5426 < GET_MODE_PRECISION (from_mode))
5427 {
5428 rtx temp1;
5429 rtx_code_label *neglabel = gen_label_rtx ();
5430
5431 /* Don't use TARGET if it isn't a register, is a hard register,
5432 or is the wrong mode. */
5433 if (!REG_P (target)
5434 || REGNO (target) < FIRST_PSEUDO_REGISTER
5435 || GET_MODE (target) != fmode)
5436 target = gen_reg_rtx (fmode);
5437
5438 imode = from_mode;
5439 do_pending_stack_adjust ();
5440
5441 /* Test whether the sign bit is set. */
5442 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5443 0, neglabel);
5444
5445 /* The sign bit is not set. Convert as signed. */
5446 expand_float (target, from, 0);
5447 emit_jump_insn (targetm.gen_jump (label));
5448 emit_barrier ();
5449
5450 /* The sign bit is set.
5451 Convert to a usable (positive signed) value by shifting right
5452 one bit, while remembering if a nonzero bit was shifted
5453 out; i.e., compute (from & 1) | (from >> 1). */
5454
5455 emit_label (neglabel);
5456 temp = expand_binop (imode, and_optab, from, const1_rtx,
5457 NULL_RTX, 1, OPTAB_LIB_WIDEN);
5458 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5459 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5460 OPTAB_LIB_WIDEN);
5461 expand_float (target, temp, 0);
5462
5463 /* Multiply by 2 to undo the shift above. */
5464 temp = expand_binop (fmode, add_optab, target, target,
5465 target, 0, OPTAB_LIB_WIDEN);
5466 if (temp != target)
5467 emit_move_insn (target, temp);
5468
5469 do_pending_stack_adjust ();
5470 emit_label (label);
5471 goto done;
5472 }
5473 }
5474
5475 /* If we are about to do some arithmetic to correct for an
5476 unsigned operand, do it in a pseudo-register. */
5477
5478 if (to_mode != fmode
5479 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5480 target = gen_reg_rtx (fmode);
5481
5482 /* Convert as signed integer to floating. */
5483 expand_float (target, from, 0);
5484
5485 /* If FROM is negative (and therefore TO is negative),
5486 correct its value by 2**bitwidth. */
5487
5488 do_pending_stack_adjust ();
5489 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
5490 0, label);
5491
5492
5493 real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
5494 temp = expand_binop (fmode, add_optab, target,
5495 const_double_from_real_value (offset, fmode),
5496 target, 0, OPTAB_LIB_WIDEN);
5497 if (temp != target)
5498 emit_move_insn (target, temp);
5499
5500 do_pending_stack_adjust ();
5501 emit_label (label);
5502 goto done;
5503 }
5504
5505 /* No hardware instruction available; call a library routine. */
5506 {
5507 rtx libfunc;
5508 rtx_insn *insns;
5509 rtx value;
5510 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5511
5512 if (is_narrower_int_mode (GET_MODE (from), SImode))
5513 from = convert_to_mode (SImode, from, unsignedp);
5514
5515 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5516 gcc_assert (libfunc);
5517
5518 start_sequence ();
5519
5520 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5521 GET_MODE (to), from, GET_MODE (from));
5522 insns = get_insns ();
5523 end_sequence ();
5524
5525 emit_libcall_block (insns, target, value,
5526 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5527 GET_MODE (to), from));
5528 }
5529
5530 done:
5531
5532 /* Copy result to requested destination
5533 if we have been computing in a temp location. */
5534
5535 if (target != to)
5536 {
5537 if (GET_MODE (target) == GET_MODE (to))
5538 emit_move_insn (to, target);
5539 else
5540 convert_move (to, target, 0);
5541 }
5542 }
5543
5544 /* Generate code to convert FROM to fixed point and store in TO. FROM
5545 must be floating point. */
5546
5547 void
expand_fix(rtx to,rtx from,int unsignedp)5548 expand_fix (rtx to, rtx from, int unsignedp)
5549 {
5550 enum insn_code icode;
5551 rtx target = to;
5552 machine_mode fmode, imode;
5553 opt_scalar_mode fmode_iter;
5554 bool must_trunc = false;
5555
5556 /* We first try to find a pair of modes, one real and one integer, at
5557 least as wide as FROM and TO, respectively, in which we can open-code
5558 this conversion. If the integer mode is wider than the mode of TO,
5559 we can do the conversion either signed or unsigned. */
5560
5561 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5562 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5563 {
5564 int doing_unsigned = unsignedp;
5565
5566 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5567 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5568 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5569
5570 if (icode != CODE_FOR_nothing)
5571 {
5572 rtx_insn *last = get_last_insn ();
5573 rtx from1 = from;
5574 if (fmode != GET_MODE (from))
5575 from1 = convert_to_mode (fmode, from, 0);
5576
5577 if (must_trunc)
5578 {
5579 rtx temp = gen_reg_rtx (GET_MODE (from1));
5580 from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
5581 temp, 0);
5582 }
5583
5584 if (imode != GET_MODE (to))
5585 target = gen_reg_rtx (imode);
5586
5587 if (maybe_emit_unop_insn (icode, target, from1,
5588 doing_unsigned ? UNSIGNED_FIX : FIX))
5589 {
5590 if (target != to)
5591 convert_move (to, target, unsignedp);
5592 return;
5593 }
5594 delete_insns_since (last);
5595 }
5596 }
5597
5598 /* For an unsigned conversion, there is one more way to do it.
5599 If we have a signed conversion, we generate code that compares
5600 the real value to the largest representable positive number. If if
5601 is smaller, the conversion is done normally. Otherwise, subtract
5602 one plus the highest signed number, convert, and add it back.
5603
5604 We only need to check all real modes, since we know we didn't find
5605 anything with a wider integer mode.
5606
5607 This code used to extend FP value into mode wider than the destination.
5608 This is needed for decimal float modes which cannot accurately
5609 represent one plus the highest signed number of the same size, but
5610 not for binary modes. Consider, for instance conversion from SFmode
5611 into DImode.
5612
5613 The hot path through the code is dealing with inputs smaller than 2^63
5614 and doing just the conversion, so there is no bits to lose.
5615
5616 In the other path we know the value is positive in the range 2^63..2^64-1
5617 inclusive. (as for other input overflow happens and result is undefined)
5618 So we know that the most important bit set in mantissa corresponds to
5619 2^63. The subtraction of 2^63 should not generate any rounding as it
5620 simply clears out that bit. The rest is trivial. */
5621
5622 scalar_int_mode to_mode;
5623 if (unsignedp
5624 && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
5625 && HWI_COMPUTABLE_MODE_P (to_mode))
5626 FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
5627 {
5628 scalar_mode fmode = fmode_iter.require ();
5629 if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
5630 0, &must_trunc)
5631 && (!DECIMAL_FLOAT_MODE_P (fmode)
5632 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
5633 {
5634 int bitsize;
5635 REAL_VALUE_TYPE offset;
5636 rtx limit;
5637 rtx_code_label *lab1, *lab2;
5638 rtx_insn *insn;
5639
5640 bitsize = GET_MODE_PRECISION (to_mode);
5641 real_2expN (&offset, bitsize - 1, fmode);
5642 limit = const_double_from_real_value (offset, fmode);
5643 lab1 = gen_label_rtx ();
5644 lab2 = gen_label_rtx ();
5645
5646 if (fmode != GET_MODE (from))
5647 from = convert_to_mode (fmode, from, 0);
5648
5649 /* See if we need to do the subtraction. */
5650 do_pending_stack_adjust ();
5651 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
5652 GET_MODE (from), 0, lab1);
5653
5654 /* If not, do the signed "fix" and branch around fixup code. */
5655 expand_fix (to, from, 0);
5656 emit_jump_insn (targetm.gen_jump (lab2));
5657 emit_barrier ();
5658
5659 /* Otherwise, subtract 2**(N-1), convert to signed number,
5660 then add 2**(N-1). Do the addition using XOR since this
5661 will often generate better code. */
5662 emit_label (lab1);
5663 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5664 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5665 expand_fix (to, target, 0);
5666 target = expand_binop (to_mode, xor_optab, to,
5667 gen_int_mode
5668 (HOST_WIDE_INT_1 << (bitsize - 1),
5669 to_mode),
5670 to, 1, OPTAB_LIB_WIDEN);
5671
5672 if (target != to)
5673 emit_move_insn (to, target);
5674
5675 emit_label (lab2);
5676
5677 if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
5678 {
5679 /* Make a place for a REG_NOTE and add it. */
5680 insn = emit_move_insn (to, to);
5681 set_dst_reg_note (insn, REG_EQUAL,
5682 gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5683 copy_rtx (from)),
5684 to);
5685 }
5686
5687 return;
5688 }
5689 }
5690
5691 /* We can't do it with an insn, so use a library call. But first ensure
5692 that the mode of TO is at least as wide as SImode, since those are the
5693 only library calls we know about. */
5694
5695 if (is_narrower_int_mode (GET_MODE (to), SImode))
5696 {
5697 target = gen_reg_rtx (SImode);
5698
5699 expand_fix (target, from, unsignedp);
5700 }
5701 else
5702 {
5703 rtx_insn *insns;
5704 rtx value;
5705 rtx libfunc;
5706
5707 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5708 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5709 gcc_assert (libfunc);
5710
5711 start_sequence ();
5712
5713 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5714 GET_MODE (to), from, GET_MODE (from));
5715 insns = get_insns ();
5716 end_sequence ();
5717
5718 emit_libcall_block (insns, target, value,
5719 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5720 GET_MODE (to), from));
5721 }
5722
5723 if (target != to)
5724 {
5725 if (GET_MODE (to) == GET_MODE (target))
5726 emit_move_insn (to, target);
5727 else
5728 convert_move (to, target, 0);
5729 }
5730 }
5731
5732
5733 /* Promote integer arguments for a libcall if necessary.
5734 emit_library_call_value cannot do the promotion because it does not
5735 know if it should do a signed or unsigned promotion. This is because
5736 there are no tree types defined for libcalls. */
5737
5738 static rtx
prepare_libcall_arg(rtx arg,int uintp)5739 prepare_libcall_arg (rtx arg, int uintp)
5740 {
5741 scalar_int_mode mode;
5742 machine_mode arg_mode;
5743 if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
5744 {
5745 /* If we need to promote the integer function argument we need to do
5746 it here instead of inside emit_library_call_value because in
5747 emit_library_call_value we don't know if we should do a signed or
5748 unsigned promotion. */
5749
5750 int unsigned_p = 0;
5751 arg_mode = promote_function_mode (NULL_TREE, mode,
5752 &unsigned_p, NULL_TREE, 0);
5753 if (arg_mode != mode)
5754 return convert_to_mode (arg_mode, arg, uintp);
5755 }
5756 return arg;
5757 }
5758
5759 /* Generate code to convert FROM or TO a fixed-point.
5760 If UINTP is true, either TO or FROM is an unsigned integer.
5761 If SATP is true, we need to saturate the result. */
5762
5763 void
expand_fixed_convert(rtx to,rtx from,int uintp,int satp)5764 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5765 {
5766 machine_mode to_mode = GET_MODE (to);
5767 machine_mode from_mode = GET_MODE (from);
5768 convert_optab tab;
5769 enum rtx_code this_code;
5770 enum insn_code code;
5771 rtx_insn *insns;
5772 rtx value;
5773 rtx libfunc;
5774
5775 if (to_mode == from_mode)
5776 {
5777 emit_move_insn (to, from);
5778 return;
5779 }
5780
5781 if (uintp)
5782 {
5783 tab = satp ? satfractuns_optab : fractuns_optab;
5784 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5785 }
5786 else
5787 {
5788 tab = satp ? satfract_optab : fract_optab;
5789 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5790 }
5791 code = convert_optab_handler (tab, to_mode, from_mode);
5792 if (code != CODE_FOR_nothing)
5793 {
5794 emit_unop_insn (code, to, from, this_code);
5795 return;
5796 }
5797
5798 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5799 gcc_assert (libfunc);
5800
5801 from = prepare_libcall_arg (from, uintp);
5802 from_mode = GET_MODE (from);
5803
5804 start_sequence ();
5805 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5806 from, from_mode);
5807 insns = get_insns ();
5808 end_sequence ();
5809
5810 emit_libcall_block (insns, to, value,
5811 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5812 }
5813
5814 /* Generate code to convert FROM to fixed point and store in TO. FROM
5815 must be floating point, TO must be signed. Use the conversion optab
5816 TAB to do the conversion. */
5817
5818 bool
expand_sfix_optab(rtx to,rtx from,convert_optab tab)5819 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5820 {
5821 enum insn_code icode;
5822 rtx target = to;
5823 machine_mode fmode, imode;
5824
5825 /* We first try to find a pair of modes, one real and one integer, at
5826 least as wide as FROM and TO, respectively, in which we can open-code
5827 this conversion. If the integer mode is wider than the mode of TO,
5828 we can do the conversion either signed or unsigned. */
5829
5830 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5831 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5832 {
5833 icode = convert_optab_handler (tab, imode, fmode);
5834 if (icode != CODE_FOR_nothing)
5835 {
5836 rtx_insn *last = get_last_insn ();
5837 if (fmode != GET_MODE (from))
5838 from = convert_to_mode (fmode, from, 0);
5839
5840 if (imode != GET_MODE (to))
5841 target = gen_reg_rtx (imode);
5842
5843 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5844 {
5845 delete_insns_since (last);
5846 continue;
5847 }
5848 if (target != to)
5849 convert_move (to, target, 0);
5850 return true;
5851 }
5852 }
5853
5854 return false;
5855 }
5856
5857 /* Report whether we have an instruction to perform the operation
5858 specified by CODE on operands of mode MODE. */
5859 int
have_insn_for(enum rtx_code code,machine_mode mode)5860 have_insn_for (enum rtx_code code, machine_mode mode)
5861 {
5862 return (code_to_optab (code)
5863 && (optab_handler (code_to_optab (code), mode)
5864 != CODE_FOR_nothing));
5865 }
5866
5867 /* Print information about the current contents of the optabs on
5868 STDERR. */
5869
5870 DEBUG_FUNCTION void
debug_optab_libfuncs(void)5871 debug_optab_libfuncs (void)
5872 {
5873 int i, j, k;
5874
5875 /* Dump the arithmetic optabs. */
5876 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
5877 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5878 {
5879 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
5880 if (l)
5881 {
5882 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5883 fprintf (stderr, "%s\t%s:\t%s\n",
5884 GET_RTX_NAME (optab_to_code ((optab) i)),
5885 GET_MODE_NAME (j),
5886 XSTR (l, 0));
5887 }
5888 }
5889
5890 /* Dump the conversion optabs. */
5891 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
5892 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5893 for (k = 0; k < NUM_MACHINE_MODES; ++k)
5894 {
5895 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
5896 (machine_mode) k);
5897 if (l)
5898 {
5899 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5900 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
5901 GET_RTX_NAME (optab_to_code ((optab) i)),
5902 GET_MODE_NAME (j),
5903 GET_MODE_NAME (k),
5904 XSTR (l, 0));
5905 }
5906 }
5907 }
5908
5909 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5910 CODE. Return 0 on failure. */
5911
5912 rtx_insn *
gen_cond_trap(enum rtx_code code,rtx op1,rtx op2,rtx tcode)5913 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
5914 {
5915 machine_mode mode = GET_MODE (op1);
5916 enum insn_code icode;
5917 rtx_insn *insn;
5918 rtx trap_rtx;
5919
5920 if (mode == VOIDmode)
5921 return 0;
5922
5923 icode = optab_handler (ctrap_optab, mode);
5924 if (icode == CODE_FOR_nothing)
5925 return 0;
5926
5927 /* Some targets only accept a zero trap code. */
5928 if (!insn_operand_matches (icode, 3, tcode))
5929 return 0;
5930
5931 do_pending_stack_adjust ();
5932 start_sequence ();
5933 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
5934 &trap_rtx, &mode);
5935 if (!trap_rtx)
5936 insn = NULL;
5937 else
5938 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
5939 tcode);
5940
5941 /* If that failed, then give up. */
5942 if (insn == 0)
5943 {
5944 end_sequence ();
5945 return 0;
5946 }
5947
5948 emit_insn (insn);
5949 insn = get_insns ();
5950 end_sequence ();
5951 return insn;
5952 }
5953
5954 /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
5955 or unsigned operation code. */
5956
5957 enum rtx_code
get_rtx_code_1(enum tree_code tcode,bool unsignedp)5958 get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
5959 {
5960 enum rtx_code code;
5961 switch (tcode)
5962 {
5963 case EQ_EXPR:
5964 code = EQ;
5965 break;
5966 case NE_EXPR:
5967 code = NE;
5968 break;
5969 case LT_EXPR:
5970 code = unsignedp ? LTU : LT;
5971 break;
5972 case LE_EXPR:
5973 code = unsignedp ? LEU : LE;
5974 break;
5975 case GT_EXPR:
5976 code = unsignedp ? GTU : GT;
5977 break;
5978 case GE_EXPR:
5979 code = unsignedp ? GEU : GE;
5980 break;
5981
5982 case UNORDERED_EXPR:
5983 code = UNORDERED;
5984 break;
5985 case ORDERED_EXPR:
5986 code = ORDERED;
5987 break;
5988 case UNLT_EXPR:
5989 code = UNLT;
5990 break;
5991 case UNLE_EXPR:
5992 code = UNLE;
5993 break;
5994 case UNGT_EXPR:
5995 code = UNGT;
5996 break;
5997 case UNGE_EXPR:
5998 code = UNGE;
5999 break;
6000 case UNEQ_EXPR:
6001 code = UNEQ;
6002 break;
6003 case LTGT_EXPR:
6004 code = LTGT;
6005 break;
6006
6007 case BIT_AND_EXPR:
6008 code = AND;
6009 break;
6010
6011 case BIT_IOR_EXPR:
6012 code = IOR;
6013 break;
6014
6015 default:
6016 code = UNKNOWN;
6017 break;
6018 }
6019 return code;
6020 }
6021
6022 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6023 or unsigned operation code. */
6024
6025 enum rtx_code
get_rtx_code(enum tree_code tcode,bool unsignedp)6026 get_rtx_code (enum tree_code tcode, bool unsignedp)
6027 {
6028 enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6029 gcc_assert (code != UNKNOWN);
6030 return code;
6031 }
6032
6033 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6034 select signed or unsigned operators. OPNO holds the index of the
6035 first comparison operand for insn ICODE. Do not generate the
6036 compare instruction itself. */
6037
6038 rtx
vector_compare_rtx(machine_mode cmp_mode,enum tree_code tcode,tree t_op0,tree t_op1,bool unsignedp,enum insn_code icode,unsigned int opno)6039 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6040 tree t_op0, tree t_op1, bool unsignedp,
6041 enum insn_code icode, unsigned int opno)
6042 {
6043 class expand_operand ops[2];
6044 rtx rtx_op0, rtx_op1;
6045 machine_mode m0, m1;
6046 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6047
6048 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6049
6050 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6051 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6052 cases, use the original mode. */
6053 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6054 EXPAND_STACK_PARM);
6055 m0 = GET_MODE (rtx_op0);
6056 if (m0 == VOIDmode)
6057 m0 = TYPE_MODE (TREE_TYPE (t_op0));
6058
6059 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6060 EXPAND_STACK_PARM);
6061 m1 = GET_MODE (rtx_op1);
6062 if (m1 == VOIDmode)
6063 m1 = TYPE_MODE (TREE_TYPE (t_op1));
6064
6065 create_input_operand (&ops[0], rtx_op0, m0);
6066 create_input_operand (&ops[1], rtx_op1, m1);
6067 if (!maybe_legitimize_operands (icode, opno, 2, ops))
6068 gcc_unreachable ();
6069 return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6070 }
6071
6072 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6073 the first vec_perm operand, assuming the second operand (for left shift
6074 first operand) is a constant vector of zeros. Return the shift distance
6075 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6076 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6077 shift or vec_shl_optab for left shift. */
6078 static rtx
shift_amt_for_vec_perm_mask(machine_mode mode,const vec_perm_indices & sel,optab shift_optab)6079 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6080 optab shift_optab)
6081 {
6082 unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6083 poly_int64 first = sel[0];
6084 if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6085 return NULL_RTX;
6086
6087 if (shift_optab == vec_shl_optab)
6088 {
6089 unsigned int nelt;
6090 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6091 return NULL_RTX;
6092 unsigned firstidx = 0;
6093 for (unsigned int i = 0; i < nelt; i++)
6094 {
6095 if (known_eq (sel[i], nelt))
6096 {
6097 if (i == 0 || firstidx)
6098 return NULL_RTX;
6099 firstidx = i;
6100 }
6101 else if (firstidx
6102 ? maybe_ne (sel[i], nelt + i - firstidx)
6103 : maybe_ge (sel[i], nelt))
6104 return NULL_RTX;
6105 }
6106
6107 if (firstidx == 0)
6108 return NULL_RTX;
6109 first = firstidx;
6110 }
6111 else if (!sel.series_p (0, 1, first, 1))
6112 {
6113 unsigned int nelt;
6114 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6115 return NULL_RTX;
6116 for (unsigned int i = 1; i < nelt; i++)
6117 {
6118 poly_int64 expected = i + first;
6119 /* Indices into the second vector are all equivalent. */
6120 if (maybe_lt (sel[i], nelt)
6121 ? maybe_ne (sel[i], expected)
6122 : maybe_lt (expected, nelt))
6123 return NULL_RTX;
6124 }
6125 }
6126
6127 return gen_int_shift_amount (mode, first * bitsize);
6128 }
6129
6130 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6131
6132 static rtx
expand_vec_perm_1(enum insn_code icode,rtx target,rtx v0,rtx v1,rtx sel)6133 expand_vec_perm_1 (enum insn_code icode, rtx target,
6134 rtx v0, rtx v1, rtx sel)
6135 {
6136 machine_mode tmode = GET_MODE (target);
6137 machine_mode smode = GET_MODE (sel);
6138 class expand_operand ops[4];
6139
6140 gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6141 || related_int_vector_mode (tmode).require () == smode);
6142 create_output_operand (&ops[0], target, tmode);
6143 create_input_operand (&ops[3], sel, smode);
6144
6145 /* Make an effort to preserve v0 == v1. The target expander is able to
6146 rely on this to determine if we're permuting a single input operand. */
6147 if (rtx_equal_p (v0, v1))
6148 {
6149 if (!insn_operand_matches (icode, 1, v0))
6150 v0 = force_reg (tmode, v0);
6151 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6152 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6153
6154 create_fixed_operand (&ops[1], v0);
6155 create_fixed_operand (&ops[2], v0);
6156 }
6157 else
6158 {
6159 create_input_operand (&ops[1], v0, tmode);
6160 create_input_operand (&ops[2], v1, tmode);
6161 }
6162
6163 if (maybe_expand_insn (icode, 4, ops))
6164 return ops[0].value;
6165 return NULL_RTX;
6166 }
6167
6168 /* Implement a permutation of vectors v0 and v1 using the permutation
6169 vector in SEL and return the result. Use TARGET to hold the result
6170 if nonnull and convenient.
6171
6172 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6173 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6174 to have a particular mode. */
6175
6176 rtx
expand_vec_perm_const(machine_mode mode,rtx v0,rtx v1,const vec_perm_builder & sel,machine_mode sel_mode,rtx target)6177 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6178 const vec_perm_builder &sel, machine_mode sel_mode,
6179 rtx target)
6180 {
6181 if (!target || !register_operand (target, mode))
6182 target = gen_reg_rtx (mode);
6183
6184 /* Set QIMODE to a different vector mode with byte elements.
6185 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6186 machine_mode qimode;
6187 if (!qimode_for_vec_perm (mode).exists (&qimode))
6188 qimode = VOIDmode;
6189
6190 rtx_insn *last = get_last_insn ();
6191
6192 bool single_arg_p = rtx_equal_p (v0, v1);
6193 /* Always specify two input vectors here and leave the target to handle
6194 cases in which the inputs are equal. Not all backends can cope with
6195 the single-input representation when testing for a double-input
6196 target instruction. */
6197 vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
6198
6199 /* See if this can be handled with a vec_shr or vec_shl. We only do this
6200 if the second (for vec_shr) or first (for vec_shl) vector is all
6201 zeroes. */
6202 insn_code shift_code = CODE_FOR_nothing;
6203 insn_code shift_code_qi = CODE_FOR_nothing;
6204 optab shift_optab = unknown_optab;
6205 rtx v2 = v0;
6206 if (v1 == CONST0_RTX (GET_MODE (v1)))
6207 shift_optab = vec_shr_optab;
6208 else if (v0 == CONST0_RTX (GET_MODE (v0)))
6209 {
6210 shift_optab = vec_shl_optab;
6211 v2 = v1;
6212 }
6213 if (shift_optab != unknown_optab)
6214 {
6215 shift_code = optab_handler (shift_optab, mode);
6216 shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6217 ? optab_handler (shift_optab, qimode)
6218 : CODE_FOR_nothing);
6219 }
6220 if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6221 {
6222 rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
6223 if (shift_amt)
6224 {
6225 class expand_operand ops[3];
6226 if (shift_amt == const0_rtx)
6227 return v2;
6228 if (shift_code != CODE_FOR_nothing)
6229 {
6230 create_output_operand (&ops[0], target, mode);
6231 create_input_operand (&ops[1], v2, mode);
6232 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6233 if (maybe_expand_insn (shift_code, 3, ops))
6234 return ops[0].value;
6235 }
6236 if (shift_code_qi != CODE_FOR_nothing)
6237 {
6238 rtx tmp = gen_reg_rtx (qimode);
6239 create_output_operand (&ops[0], tmp, qimode);
6240 create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
6241 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6242 if (maybe_expand_insn (shift_code_qi, 3, ops))
6243 return gen_lowpart (mode, ops[0].value);
6244 }
6245 }
6246 }
6247
6248 if (targetm.vectorize.vec_perm_const != NULL)
6249 {
6250 if (single_arg_p)
6251 v1 = v0;
6252
6253 if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
6254 return target;
6255 }
6256
6257 /* Fall back to a constant byte-based permutation. */
6258 vec_perm_indices qimode_indices;
6259 rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6260 if (qimode != VOIDmode)
6261 {
6262 qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6263 target_qi = gen_reg_rtx (qimode);
6264 v0_qi = gen_lowpart (qimode, v0);
6265 v1_qi = gen_lowpart (qimode, v1);
6266 if (targetm.vectorize.vec_perm_const != NULL
6267 && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
6268 v1_qi, qimode_indices))
6269 return gen_lowpart (mode, target_qi);
6270 }
6271
6272 v0 = force_reg (mode, v0);
6273 if (single_arg_p)
6274 v1 = v0;
6275 v1 = force_reg (mode, v1);
6276
6277 /* Otherwise expand as a fully variable permuation. */
6278
6279 /* The optabs are only defined for selectors with the same width
6280 as the values being permuted. */
6281 machine_mode required_sel_mode;
6282 if (!related_int_vector_mode (mode).exists (&required_sel_mode))
6283 {
6284 delete_insns_since (last);
6285 return NULL_RTX;
6286 }
6287
6288 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6289 If that isn't the mode we want then we need to prove that using
6290 REQUIRED_SEL_MODE is OK. */
6291 if (sel_mode != required_sel_mode)
6292 {
6293 if (!selector_fits_mode_p (required_sel_mode, indices))
6294 {
6295 delete_insns_since (last);
6296 return NULL_RTX;
6297 }
6298 sel_mode = required_sel_mode;
6299 }
6300
6301 insn_code icode = direct_optab_handler (vec_perm_optab, mode);
6302 if (icode != CODE_FOR_nothing)
6303 {
6304 rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6305 rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
6306 if (tmp)
6307 return tmp;
6308 }
6309
6310 if (qimode != VOIDmode
6311 && selector_fits_mode_p (qimode, qimode_indices))
6312 {
6313 icode = direct_optab_handler (vec_perm_optab, qimode);
6314 if (icode != CODE_FOR_nothing)
6315 {
6316 rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6317 rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
6318 if (tmp)
6319 return gen_lowpart (mode, tmp);
6320 }
6321 }
6322
6323 delete_insns_since (last);
6324 return NULL_RTX;
6325 }
6326
6327 /* Implement a permutation of vectors v0 and v1 using the permutation
6328 vector in SEL and return the result. Use TARGET to hold the result
6329 if nonnull and convenient.
6330
6331 MODE is the mode of the vectors being permuted (V0 and V1).
6332 SEL must have the integer equivalent of MODE and is known to be
6333 unsuitable for permutes with a constant permutation vector. */
6334
6335 rtx
expand_vec_perm_var(machine_mode mode,rtx v0,rtx v1,rtx sel,rtx target)6336 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6337 {
6338 enum insn_code icode;
6339 unsigned int i, u;
6340 rtx tmp, sel_qi;
6341
6342 u = GET_MODE_UNIT_SIZE (mode);
6343
6344 if (!target || GET_MODE (target) != mode)
6345 target = gen_reg_rtx (mode);
6346
6347 icode = direct_optab_handler (vec_perm_optab, mode);
6348 if (icode != CODE_FOR_nothing)
6349 {
6350 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6351 if (tmp)
6352 return tmp;
6353 }
6354
6355 /* As a special case to aid several targets, lower the element-based
6356 permutation to a byte-based permutation and try again. */
6357 machine_mode qimode;
6358 if (!qimode_for_vec_perm (mode).exists (&qimode)
6359 || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
6360 return NULL_RTX;
6361 icode = direct_optab_handler (vec_perm_optab, qimode);
6362 if (icode == CODE_FOR_nothing)
6363 return NULL_RTX;
6364
6365 /* Multiply each element by its byte size. */
6366 machine_mode selmode = GET_MODE (sel);
6367 if (u == 2)
6368 sel = expand_simple_binop (selmode, PLUS, sel, sel,
6369 NULL, 0, OPTAB_DIRECT);
6370 else
6371 sel = expand_simple_binop (selmode, ASHIFT, sel,
6372 gen_int_shift_amount (selmode, exact_log2 (u)),
6373 NULL, 0, OPTAB_DIRECT);
6374 gcc_assert (sel != NULL);
6375
6376 /* Broadcast the low byte each element into each of its bytes.
6377 The encoding has U interleaved stepped patterns, one for each
6378 byte of an element. */
6379 vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
6380 unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
6381 for (i = 0; i < 3; ++i)
6382 for (unsigned int j = 0; j < u; ++j)
6383 const_sel.quick_push (i * u + low_byte_in_u);
6384 sel = gen_lowpart (qimode, sel);
6385 sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
6386 gcc_assert (sel != NULL);
6387
6388 /* Add the byte offset to each byte element. */
6389 /* Note that the definition of the indicies here is memory ordering,
6390 so there should be no difference between big and little endian. */
6391 rtx_vector_builder byte_indices (qimode, u, 1);
6392 for (i = 0; i < u; ++i)
6393 byte_indices.quick_push (GEN_INT (i));
6394 tmp = byte_indices.build ();
6395 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6396 sel, 0, OPTAB_DIRECT);
6397 gcc_assert (sel_qi != NULL);
6398
6399 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6400 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6401 gen_lowpart (qimode, v1), sel_qi);
6402 if (tmp)
6403 tmp = gen_lowpart (mode, tmp);
6404 return tmp;
6405 }
6406
6407 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6408 Use TARGET for the result if nonnull and convenient. */
6409
6410 rtx
expand_vec_series_expr(machine_mode vmode,rtx op0,rtx op1,rtx target)6411 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
6412 {
6413 class expand_operand ops[3];
6414 enum insn_code icode;
6415 machine_mode emode = GET_MODE_INNER (vmode);
6416
6417 icode = direct_optab_handler (vec_series_optab, vmode);
6418 gcc_assert (icode != CODE_FOR_nothing);
6419
6420 create_output_operand (&ops[0], target, vmode);
6421 create_input_operand (&ops[1], op0, emode);
6422 create_input_operand (&ops[2], op1, emode);
6423
6424 expand_insn (icode, 3, ops);
6425 return ops[0].value;
6426 }
6427
6428 /* Generate insns for a vector comparison into a mask. */
6429
6430 rtx
expand_vec_cmp_expr(tree type,tree exp,rtx target)6431 expand_vec_cmp_expr (tree type, tree exp, rtx target)
6432 {
6433 class expand_operand ops[4];
6434 enum insn_code icode;
6435 rtx comparison;
6436 machine_mode mask_mode = TYPE_MODE (type);
6437 machine_mode vmode;
6438 bool unsignedp;
6439 tree op0a, op0b;
6440 enum tree_code tcode;
6441
6442 op0a = TREE_OPERAND (exp, 0);
6443 op0b = TREE_OPERAND (exp, 1);
6444 tcode = TREE_CODE (exp);
6445
6446 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6447 vmode = TYPE_MODE (TREE_TYPE (op0a));
6448
6449 icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
6450 if (icode == CODE_FOR_nothing)
6451 {
6452 if (tcode == EQ_EXPR || tcode == NE_EXPR)
6453 icode = get_vec_cmp_eq_icode (vmode, mask_mode);
6454 if (icode == CODE_FOR_nothing)
6455 return 0;
6456 }
6457
6458 comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
6459 unsignedp, icode, 2);
6460 create_output_operand (&ops[0], target, mask_mode);
6461 create_fixed_operand (&ops[1], comparison);
6462 create_fixed_operand (&ops[2], XEXP (comparison, 0));
6463 create_fixed_operand (&ops[3], XEXP (comparison, 1));
6464 expand_insn (icode, 4, ops);
6465 return ops[0].value;
6466 }
6467
6468 /* Expand a highpart multiply. */
6469
6470 rtx
expand_mult_highpart(machine_mode mode,rtx op0,rtx op1,rtx target,bool uns_p)6471 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6472 rtx target, bool uns_p)
6473 {
6474 class expand_operand eops[3];
6475 enum insn_code icode;
6476 int method, i;
6477 machine_mode wmode;
6478 rtx m1, m2;
6479 optab tab1, tab2;
6480
6481 method = can_mult_highpart_p (mode, uns_p);
6482 switch (method)
6483 {
6484 case 0:
6485 return NULL_RTX;
6486 case 1:
6487 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6488 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6489 OPTAB_LIB_WIDEN);
6490 case 2:
6491 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6492 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6493 break;
6494 case 3:
6495 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6496 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6497 if (BYTES_BIG_ENDIAN)
6498 std::swap (tab1, tab2);
6499 break;
6500 default:
6501 gcc_unreachable ();
6502 }
6503
6504 icode = optab_handler (tab1, mode);
6505 wmode = insn_data[icode].operand[0].mode;
6506 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
6507 GET_MODE_NUNITS (mode)));
6508 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
6509
6510 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6511 create_input_operand (&eops[1], op0, mode);
6512 create_input_operand (&eops[2], op1, mode);
6513 expand_insn (icode, 3, eops);
6514 m1 = gen_lowpart (mode, eops[0].value);
6515
6516 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6517 create_input_operand (&eops[1], op0, mode);
6518 create_input_operand (&eops[2], op1, mode);
6519 expand_insn (optab_handler (tab2, mode), 3, eops);
6520 m2 = gen_lowpart (mode, eops[0].value);
6521
6522 vec_perm_builder sel;
6523 if (method == 2)
6524 {
6525 /* The encoding has 2 interleaved stepped patterns. */
6526 sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
6527 for (i = 0; i < 6; ++i)
6528 sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
6529 + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
6530 }
6531 else
6532 {
6533 /* The encoding has a single interleaved stepped pattern. */
6534 sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
6535 for (i = 0; i < 3; ++i)
6536 sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6537 }
6538
6539 return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
6540 }
6541
6542 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6543 pattern. */
6544
6545 static void
find_cc_set(rtx x,const_rtx pat,void * data)6546 find_cc_set (rtx x, const_rtx pat, void *data)
6547 {
6548 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6549 && GET_CODE (pat) == SET)
6550 {
6551 rtx *p_cc_reg = (rtx *) data;
6552 gcc_assert (!*p_cc_reg);
6553 *p_cc_reg = x;
6554 }
6555 }
6556
6557 /* This is a helper function for the other atomic operations. This function
6558 emits a loop that contains SEQ that iterates until a compare-and-swap
6559 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6560 a set of instructions that takes a value from OLD_REG as an input and
6561 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6562 set to the current contents of MEM. After SEQ, a compare-and-swap will
6563 attempt to update MEM with NEW_REG. The function returns true when the
6564 loop was generated successfully. */
6565
6566 static bool
expand_compare_and_swap_loop(rtx mem,rtx old_reg,rtx new_reg,rtx seq)6567 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6568 {
6569 machine_mode mode = GET_MODE (mem);
6570 rtx_code_label *label;
6571 rtx cmp_reg, success, oldval;
6572
6573 /* The loop we want to generate looks like
6574
6575 cmp_reg = mem;
6576 label:
6577 old_reg = cmp_reg;
6578 seq;
6579 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6580 if (success)
6581 goto label;
6582
6583 Note that we only do the plain load from memory once. Subsequent
6584 iterations use the value loaded by the compare-and-swap pattern. */
6585
6586 label = gen_label_rtx ();
6587 cmp_reg = gen_reg_rtx (mode);
6588
6589 emit_move_insn (cmp_reg, mem);
6590 emit_label (label);
6591 emit_move_insn (old_reg, cmp_reg);
6592 if (seq)
6593 emit_insn (seq);
6594
6595 success = NULL_RTX;
6596 oldval = cmp_reg;
6597 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6598 new_reg, false, MEMMODEL_SYNC_SEQ_CST,
6599 MEMMODEL_RELAXED))
6600 return false;
6601
6602 if (oldval != cmp_reg)
6603 emit_move_insn (cmp_reg, oldval);
6604
6605 /* Mark this jump predicted not taken. */
6606 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6607 GET_MODE (success), 1, label,
6608 profile_probability::guessed_never ());
6609 return true;
6610 }
6611
6612
6613 /* This function tries to emit an atomic_exchange intruction. VAL is written
6614 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6615 using TARGET if possible. */
6616
6617 static rtx
maybe_emit_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6618 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6619 {
6620 machine_mode mode = GET_MODE (mem);
6621 enum insn_code icode;
6622
6623 /* If the target supports the exchange directly, great. */
6624 icode = direct_optab_handler (atomic_exchange_optab, mode);
6625 if (icode != CODE_FOR_nothing)
6626 {
6627 class expand_operand ops[4];
6628
6629 create_output_operand (&ops[0], target, mode);
6630 create_fixed_operand (&ops[1], mem);
6631 create_input_operand (&ops[2], val, mode);
6632 create_integer_operand (&ops[3], model);
6633 if (maybe_expand_insn (icode, 4, ops))
6634 return ops[0].value;
6635 }
6636
6637 return NULL_RTX;
6638 }
6639
6640 /* This function tries to implement an atomic exchange operation using
6641 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6642 The previous contents of *MEM are returned, using TARGET if possible.
6643 Since this instructionn is an acquire barrier only, stronger memory
6644 models may require additional barriers to be emitted. */
6645
6646 static rtx
maybe_emit_sync_lock_test_and_set(rtx target,rtx mem,rtx val,enum memmodel model)6647 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6648 enum memmodel model)
6649 {
6650 machine_mode mode = GET_MODE (mem);
6651 enum insn_code icode;
6652 rtx_insn *last_insn = get_last_insn ();
6653
6654 icode = optab_handler (sync_lock_test_and_set_optab, mode);
6655
6656 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6657 exists, and the memory model is stronger than acquire, add a release
6658 barrier before the instruction. */
6659
6660 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6661 expand_mem_thread_fence (model);
6662
6663 if (icode != CODE_FOR_nothing)
6664 {
6665 class expand_operand ops[3];
6666 create_output_operand (&ops[0], target, mode);
6667 create_fixed_operand (&ops[1], mem);
6668 create_input_operand (&ops[2], val, mode);
6669 if (maybe_expand_insn (icode, 3, ops))
6670 return ops[0].value;
6671 }
6672
6673 /* If an external test-and-set libcall is provided, use that instead of
6674 any external compare-and-swap that we might get from the compare-and-
6675 swap-loop expansion later. */
6676 if (!can_compare_and_swap_p (mode, false))
6677 {
6678 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6679 if (libfunc != NULL)
6680 {
6681 rtx addr;
6682
6683 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6684 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6685 mode, addr, ptr_mode,
6686 val, mode);
6687 }
6688 }
6689
6690 /* If the test_and_set can't be emitted, eliminate any barrier that might
6691 have been emitted. */
6692 delete_insns_since (last_insn);
6693 return NULL_RTX;
6694 }
6695
6696 /* This function tries to implement an atomic exchange operation using a
6697 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6698 *MEM are returned, using TARGET if possible. No memory model is required
6699 since a compare_and_swap loop is seq-cst. */
6700
6701 static rtx
maybe_emit_compare_and_swap_exchange_loop(rtx target,rtx mem,rtx val)6702 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6703 {
6704 machine_mode mode = GET_MODE (mem);
6705
6706 if (can_compare_and_swap_p (mode, true))
6707 {
6708 if (!target || !register_operand (target, mode))
6709 target = gen_reg_rtx (mode);
6710 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
6711 return target;
6712 }
6713
6714 return NULL_RTX;
6715 }
6716
6717 /* This function tries to implement an atomic test-and-set operation
6718 using the atomic_test_and_set instruction pattern. A boolean value
6719 is returned from the operation, using TARGET if possible. */
6720
6721 static rtx
maybe_emit_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6722 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6723 {
6724 machine_mode pat_bool_mode;
6725 class expand_operand ops[3];
6726
6727 if (!targetm.have_atomic_test_and_set ())
6728 return NULL_RTX;
6729
6730 /* While we always get QImode from __atomic_test_and_set, we get
6731 other memory modes from __sync_lock_test_and_set. Note that we
6732 use no endian adjustment here. This matches the 4.6 behavior
6733 in the Sparc backend. */
6734 enum insn_code icode = targetm.code_for_atomic_test_and_set;
6735 gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6736 if (GET_MODE (mem) != QImode)
6737 mem = adjust_address_nv (mem, QImode, 0);
6738
6739 pat_bool_mode = insn_data[icode].operand[0].mode;
6740 create_output_operand (&ops[0], target, pat_bool_mode);
6741 create_fixed_operand (&ops[1], mem);
6742 create_integer_operand (&ops[2], model);
6743
6744 if (maybe_expand_insn (icode, 3, ops))
6745 return ops[0].value;
6746 return NULL_RTX;
6747 }
6748
6749 /* This function expands the legacy _sync_lock test_and_set operation which is
6750 generally an atomic exchange. Some limited targets only allow the
6751 constant 1 to be stored. This is an ACQUIRE operation.
6752
6753 TARGET is an optional place to stick the return value.
6754 MEM is where VAL is stored. */
6755
6756 rtx
expand_sync_lock_test_and_set(rtx target,rtx mem,rtx val)6757 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
6758 {
6759 rtx ret;
6760
6761 /* Try an atomic_exchange first. */
6762 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
6763 if (ret)
6764 return ret;
6765
6766 ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
6767 MEMMODEL_SYNC_ACQUIRE);
6768 if (ret)
6769 return ret;
6770
6771 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6772 if (ret)
6773 return ret;
6774
6775 /* If there are no other options, try atomic_test_and_set if the value
6776 being stored is 1. */
6777 if (val == const1_rtx)
6778 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
6779
6780 return ret;
6781 }
6782
6783 /* This function expands the atomic test_and_set operation:
6784 atomically store a boolean TRUE into MEM and return the previous value.
6785
6786 MEMMODEL is the memory model variant to use.
6787 TARGET is an optional place to stick the return value. */
6788
6789 rtx
expand_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6790 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6791 {
6792 machine_mode mode = GET_MODE (mem);
6793 rtx ret, trueval, subtarget;
6794
6795 ret = maybe_emit_atomic_test_and_set (target, mem, model);
6796 if (ret)
6797 return ret;
6798
6799 /* Be binary compatible with non-default settings of trueval, and different
6800 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6801 another only has atomic-exchange. */
6802 if (targetm.atomic_test_and_set_trueval == 1)
6803 {
6804 trueval = const1_rtx;
6805 subtarget = target ? target : gen_reg_rtx (mode);
6806 }
6807 else
6808 {
6809 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
6810 subtarget = gen_reg_rtx (mode);
6811 }
6812
6813 /* Try the atomic-exchange optab... */
6814 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
6815
6816 /* ... then an atomic-compare-and-swap loop ... */
6817 if (!ret)
6818 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
6819
6820 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6821 if (!ret)
6822 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
6823
6824 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6825 things with the value 1. Thus we try again without trueval. */
6826 if (!ret && targetm.atomic_test_and_set_trueval != 1)
6827 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
6828
6829 /* Failing all else, assume a single threaded environment and simply
6830 perform the operation. */
6831 if (!ret)
6832 {
6833 /* If the result is ignored skip the move to target. */
6834 if (subtarget != const0_rtx)
6835 emit_move_insn (subtarget, mem);
6836
6837 emit_move_insn (mem, trueval);
6838 ret = subtarget;
6839 }
6840
6841 /* Recall that have to return a boolean value; rectify if trueval
6842 is not exactly one. */
6843 if (targetm.atomic_test_and_set_trueval != 1)
6844 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
6845
6846 return ret;
6847 }
6848
6849 /* This function expands the atomic exchange operation:
6850 atomically store VAL in MEM and return the previous value in MEM.
6851
6852 MEMMODEL is the memory model variant to use.
6853 TARGET is an optional place to stick the return value. */
6854
6855 rtx
expand_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6856 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6857 {
6858 machine_mode mode = GET_MODE (mem);
6859 rtx ret;
6860
6861 /* If loads are not atomic for the required size and we are not called to
6862 provide a __sync builtin, do not do anything so that we stay consistent
6863 with atomic loads of the same size. */
6864 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6865 return NULL_RTX;
6866
6867 ret = maybe_emit_atomic_exchange (target, mem, val, model);
6868
6869 /* Next try a compare-and-swap loop for the exchange. */
6870 if (!ret)
6871 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6872
6873 return ret;
6874 }
6875
6876 /* This function expands the atomic compare exchange operation:
6877
6878 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6879 *PTARGET_OVAL is an optional place to store the old value from memory.
6880 Both target parameters may be NULL or const0_rtx to indicate that we do
6881 not care about that return value. Both target parameters are updated on
6882 success to the actual location of the corresponding result.
6883
6884 MEMMODEL is the memory model variant to use.
6885
6886 The return value of the function is true for success. */
6887
6888 bool
expand_atomic_compare_and_swap(rtx * ptarget_bool,rtx * ptarget_oval,rtx mem,rtx expected,rtx desired,bool is_weak,enum memmodel succ_model,enum memmodel fail_model)6889 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
6890 rtx mem, rtx expected, rtx desired,
6891 bool is_weak, enum memmodel succ_model,
6892 enum memmodel fail_model)
6893 {
6894 machine_mode mode = GET_MODE (mem);
6895 class expand_operand ops[8];
6896 enum insn_code icode;
6897 rtx target_oval, target_bool = NULL_RTX;
6898 rtx libfunc;
6899
6900 /* If loads are not atomic for the required size and we are not called to
6901 provide a __sync builtin, do not do anything so that we stay consistent
6902 with atomic loads of the same size. */
6903 if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
6904 return false;
6905
6906 /* Load expected into a register for the compare and swap. */
6907 if (MEM_P (expected))
6908 expected = copy_to_reg (expected);
6909
6910 /* Make sure we always have some place to put the return oldval.
6911 Further, make sure that place is distinct from the input expected,
6912 just in case we need that path down below. */
6913 if (ptarget_oval && *ptarget_oval == const0_rtx)
6914 ptarget_oval = NULL;
6915
6916 if (ptarget_oval == NULL
6917 || (target_oval = *ptarget_oval) == NULL
6918 || reg_overlap_mentioned_p (expected, target_oval))
6919 target_oval = gen_reg_rtx (mode);
6920
6921 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6922 if (icode != CODE_FOR_nothing)
6923 {
6924 machine_mode bool_mode = insn_data[icode].operand[0].mode;
6925
6926 if (ptarget_bool && *ptarget_bool == const0_rtx)
6927 ptarget_bool = NULL;
6928
6929 /* Make sure we always have a place for the bool operand. */
6930 if (ptarget_bool == NULL
6931 || (target_bool = *ptarget_bool) == NULL
6932 || GET_MODE (target_bool) != bool_mode)
6933 target_bool = gen_reg_rtx (bool_mode);
6934
6935 /* Emit the compare_and_swap. */
6936 create_output_operand (&ops[0], target_bool, bool_mode);
6937 create_output_operand (&ops[1], target_oval, mode);
6938 create_fixed_operand (&ops[2], mem);
6939 create_input_operand (&ops[3], expected, mode);
6940 create_input_operand (&ops[4], desired, mode);
6941 create_integer_operand (&ops[5], is_weak);
6942 create_integer_operand (&ops[6], succ_model);
6943 create_integer_operand (&ops[7], fail_model);
6944 if (maybe_expand_insn (icode, 8, ops))
6945 {
6946 /* Return success/failure. */
6947 target_bool = ops[0].value;
6948 target_oval = ops[1].value;
6949 goto success;
6950 }
6951 }
6952
6953 /* Otherwise fall back to the original __sync_val_compare_and_swap
6954 which is always seq-cst. */
6955 icode = optab_handler (sync_compare_and_swap_optab, mode);
6956 if (icode != CODE_FOR_nothing)
6957 {
6958 rtx cc_reg;
6959
6960 create_output_operand (&ops[0], target_oval, mode);
6961 create_fixed_operand (&ops[1], mem);
6962 create_input_operand (&ops[2], expected, mode);
6963 create_input_operand (&ops[3], desired, mode);
6964 if (!maybe_expand_insn (icode, 4, ops))
6965 return false;
6966
6967 target_oval = ops[0].value;
6968
6969 /* If the caller isn't interested in the boolean return value,
6970 skip the computation of it. */
6971 if (ptarget_bool == NULL)
6972 goto success;
6973
6974 /* Otherwise, work out if the compare-and-swap succeeded. */
6975 cc_reg = NULL_RTX;
6976 if (have_insn_for (COMPARE, CCmode))
6977 note_stores (get_last_insn (), find_cc_set, &cc_reg);
6978 if (cc_reg)
6979 {
6980 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
6981 const0_rtx, VOIDmode, 0, 1);
6982 goto success;
6983 }
6984 goto success_bool_from_val;
6985 }
6986
6987 /* Also check for library support for __sync_val_compare_and_swap. */
6988 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
6989 if (libfunc != NULL)
6990 {
6991 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6992 rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6993 mode, addr, ptr_mode,
6994 expected, mode, desired, mode);
6995 emit_move_insn (target_oval, target);
6996
6997 /* Compute the boolean return value only if requested. */
6998 if (ptarget_bool)
6999 goto success_bool_from_val;
7000 else
7001 goto success;
7002 }
7003
7004 /* Failure. */
7005 return false;
7006
7007 success_bool_from_val:
7008 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7009 expected, VOIDmode, 1, 1);
7010 success:
7011 /* Make sure that the oval output winds up where the caller asked. */
7012 if (ptarget_oval)
7013 *ptarget_oval = target_oval;
7014 if (ptarget_bool)
7015 *ptarget_bool = target_bool;
7016 return true;
7017 }
7018
7019 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7020
7021 static void
expand_asm_memory_blockage(void)7022 expand_asm_memory_blockage (void)
7023 {
7024 rtx asm_op, clob;
7025
7026 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7027 rtvec_alloc (0), rtvec_alloc (0),
7028 rtvec_alloc (0), UNKNOWN_LOCATION);
7029 MEM_VOLATILE_P (asm_op) = 1;
7030
7031 clob = gen_rtx_SCRATCH (VOIDmode);
7032 clob = gen_rtx_MEM (BLKmode, clob);
7033 clob = gen_rtx_CLOBBER (VOIDmode, clob);
7034
7035 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7036 }
7037
7038 /* Do not propagate memory accesses across this point. */
7039
7040 static void
expand_memory_blockage(void)7041 expand_memory_blockage (void)
7042 {
7043 if (targetm.have_memory_blockage ())
7044 emit_insn (targetm.gen_memory_blockage ());
7045 else
7046 expand_asm_memory_blockage ();
7047 }
7048
7049 /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7050 same time clobbering the register set specified by REGS. */
7051
7052 void
expand_asm_reg_clobber_mem_blockage(HARD_REG_SET regs)7053 expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7054 {
7055 rtx asm_op, clob_mem;
7056
7057 unsigned int num_of_regs = 0;
7058 for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7059 if (TEST_HARD_REG_BIT (regs, i))
7060 num_of_regs++;
7061
7062 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7063 rtvec_alloc (0), rtvec_alloc (0),
7064 rtvec_alloc (0), UNKNOWN_LOCATION);
7065 MEM_VOLATILE_P (asm_op) = 1;
7066
7067 rtvec v = rtvec_alloc (num_of_regs + 2);
7068
7069 clob_mem = gen_rtx_SCRATCH (VOIDmode);
7070 clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7071 clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7072
7073 RTVEC_ELT (v, 0) = asm_op;
7074 RTVEC_ELT (v, 1) = clob_mem;
7075
7076 if (num_of_regs > 0)
7077 {
7078 unsigned int j = 2;
7079 for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7080 if (TEST_HARD_REG_BIT (regs, i))
7081 {
7082 RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7083 j++;
7084 }
7085 gcc_assert (j == (num_of_regs + 2));
7086 }
7087
7088 emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7089 }
7090
7091 /* This routine will either emit the mem_thread_fence pattern or issue a
7092 sync_synchronize to generate a fence for memory model MEMMODEL. */
7093
7094 void
expand_mem_thread_fence(enum memmodel model)7095 expand_mem_thread_fence (enum memmodel model)
7096 {
7097 if (is_mm_relaxed (model))
7098 return;
7099 if (targetm.have_mem_thread_fence ())
7100 {
7101 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7102 expand_memory_blockage ();
7103 }
7104 else if (targetm.have_memory_barrier ())
7105 emit_insn (targetm.gen_memory_barrier ());
7106 else if (synchronize_libfunc != NULL_RTX)
7107 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
7108 else
7109 expand_memory_blockage ();
7110 }
7111
7112 /* Emit a signal fence with given memory model. */
7113
7114 void
expand_mem_signal_fence(enum memmodel model)7115 expand_mem_signal_fence (enum memmodel model)
7116 {
7117 /* No machine barrier is required to implement a signal fence, but
7118 a compiler memory barrier must be issued, except for relaxed MM. */
7119 if (!is_mm_relaxed (model))
7120 expand_memory_blockage ();
7121 }
7122
7123 /* This function expands the atomic load operation:
7124 return the atomically loaded value in MEM.
7125
7126 MEMMODEL is the memory model variant to use.
7127 TARGET is an option place to stick the return value. */
7128
7129 rtx
expand_atomic_load(rtx target,rtx mem,enum memmodel model)7130 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7131 {
7132 machine_mode mode = GET_MODE (mem);
7133 enum insn_code icode;
7134
7135 /* If the target supports the load directly, great. */
7136 icode = direct_optab_handler (atomic_load_optab, mode);
7137 if (icode != CODE_FOR_nothing)
7138 {
7139 class expand_operand ops[3];
7140 rtx_insn *last = get_last_insn ();
7141 if (is_mm_seq_cst (model))
7142 expand_memory_blockage ();
7143
7144 create_output_operand (&ops[0], target, mode);
7145 create_fixed_operand (&ops[1], mem);
7146 create_integer_operand (&ops[2], model);
7147 if (maybe_expand_insn (icode, 3, ops))
7148 {
7149 if (!is_mm_relaxed (model))
7150 expand_memory_blockage ();
7151 return ops[0].value;
7152 }
7153 delete_insns_since (last);
7154 }
7155
7156 /* If the size of the object is greater than word size on this target,
7157 then we assume that a load will not be atomic. We could try to
7158 emulate a load with a compare-and-swap operation, but the store that
7159 doing this could result in would be incorrect if this is a volatile
7160 atomic load or targetting read-only-mapped memory. */
7161 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7162 /* If there is no atomic load, leave the library call. */
7163 return NULL_RTX;
7164
7165 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7166 if (!target || target == const0_rtx)
7167 target = gen_reg_rtx (mode);
7168
7169 /* For SEQ_CST, emit a barrier before the load. */
7170 if (is_mm_seq_cst (model))
7171 expand_mem_thread_fence (model);
7172
7173 emit_move_insn (target, mem);
7174
7175 /* Emit the appropriate barrier after the load. */
7176 expand_mem_thread_fence (model);
7177
7178 return target;
7179 }
7180
7181 /* This function expands the atomic store operation:
7182 Atomically store VAL in MEM.
7183 MEMMODEL is the memory model variant to use.
7184 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7185 function returns const0_rtx if a pattern was emitted. */
7186
7187 rtx
expand_atomic_store(rtx mem,rtx val,enum memmodel model,bool use_release)7188 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7189 {
7190 machine_mode mode = GET_MODE (mem);
7191 enum insn_code icode;
7192 class expand_operand ops[3];
7193
7194 /* If the target supports the store directly, great. */
7195 icode = direct_optab_handler (atomic_store_optab, mode);
7196 if (icode != CODE_FOR_nothing)
7197 {
7198 rtx_insn *last = get_last_insn ();
7199 if (!is_mm_relaxed (model))
7200 expand_memory_blockage ();
7201 create_fixed_operand (&ops[0], mem);
7202 create_input_operand (&ops[1], val, mode);
7203 create_integer_operand (&ops[2], model);
7204 if (maybe_expand_insn (icode, 3, ops))
7205 {
7206 if (is_mm_seq_cst (model))
7207 expand_memory_blockage ();
7208 return const0_rtx;
7209 }
7210 delete_insns_since (last);
7211 }
7212
7213 /* If using __sync_lock_release is a viable alternative, try it.
7214 Note that this will not be set to true if we are expanding a generic
7215 __atomic_store_n. */
7216 if (use_release)
7217 {
7218 icode = direct_optab_handler (sync_lock_release_optab, mode);
7219 if (icode != CODE_FOR_nothing)
7220 {
7221 create_fixed_operand (&ops[0], mem);
7222 create_input_operand (&ops[1], const0_rtx, mode);
7223 if (maybe_expand_insn (icode, 2, ops))
7224 {
7225 /* lock_release is only a release barrier. */
7226 if (is_mm_seq_cst (model))
7227 expand_mem_thread_fence (model);
7228 return const0_rtx;
7229 }
7230 }
7231 }
7232
7233 /* If the size of the object is greater than word size on this target,
7234 a default store will not be atomic. */
7235 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7236 {
7237 /* If loads are atomic or we are called to provide a __sync builtin,
7238 we can try a atomic_exchange and throw away the result. Otherwise,
7239 don't do anything so that we do not create an inconsistency between
7240 loads and stores. */
7241 if (can_atomic_load_p (mode) || is_mm_sync (model))
7242 {
7243 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7244 if (!target)
7245 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7246 val);
7247 if (target)
7248 return const0_rtx;
7249 }
7250 return NULL_RTX;
7251 }
7252
7253 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7254 expand_mem_thread_fence (model);
7255
7256 emit_move_insn (mem, val);
7257
7258 /* For SEQ_CST, also emit a barrier after the store. */
7259 if (is_mm_seq_cst (model))
7260 expand_mem_thread_fence (model);
7261
7262 return const0_rtx;
7263 }
7264
7265
7266 /* Structure containing the pointers and values required to process the
7267 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7268
7269 struct atomic_op_functions
7270 {
7271 direct_optab mem_fetch_before;
7272 direct_optab mem_fetch_after;
7273 direct_optab mem_no_result;
7274 optab fetch_before;
7275 optab fetch_after;
7276 direct_optab no_result;
7277 enum rtx_code reverse_code;
7278 };
7279
7280
7281 /* Fill in structure pointed to by OP with the various optab entries for an
7282 operation of type CODE. */
7283
7284 static void
get_atomic_op_for_code(struct atomic_op_functions * op,enum rtx_code code)7285 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7286 {
7287 gcc_assert (op!= NULL);
7288
7289 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7290 in the source code during compilation, and the optab entries are not
7291 computable until runtime. Fill in the values at runtime. */
7292 switch (code)
7293 {
7294 case PLUS:
7295 op->mem_fetch_before = atomic_fetch_add_optab;
7296 op->mem_fetch_after = atomic_add_fetch_optab;
7297 op->mem_no_result = atomic_add_optab;
7298 op->fetch_before = sync_old_add_optab;
7299 op->fetch_after = sync_new_add_optab;
7300 op->no_result = sync_add_optab;
7301 op->reverse_code = MINUS;
7302 break;
7303 case MINUS:
7304 op->mem_fetch_before = atomic_fetch_sub_optab;
7305 op->mem_fetch_after = atomic_sub_fetch_optab;
7306 op->mem_no_result = atomic_sub_optab;
7307 op->fetch_before = sync_old_sub_optab;
7308 op->fetch_after = sync_new_sub_optab;
7309 op->no_result = sync_sub_optab;
7310 op->reverse_code = PLUS;
7311 break;
7312 case XOR:
7313 op->mem_fetch_before = atomic_fetch_xor_optab;
7314 op->mem_fetch_after = atomic_xor_fetch_optab;
7315 op->mem_no_result = atomic_xor_optab;
7316 op->fetch_before = sync_old_xor_optab;
7317 op->fetch_after = sync_new_xor_optab;
7318 op->no_result = sync_xor_optab;
7319 op->reverse_code = XOR;
7320 break;
7321 case AND:
7322 op->mem_fetch_before = atomic_fetch_and_optab;
7323 op->mem_fetch_after = atomic_and_fetch_optab;
7324 op->mem_no_result = atomic_and_optab;
7325 op->fetch_before = sync_old_and_optab;
7326 op->fetch_after = sync_new_and_optab;
7327 op->no_result = sync_and_optab;
7328 op->reverse_code = UNKNOWN;
7329 break;
7330 case IOR:
7331 op->mem_fetch_before = atomic_fetch_or_optab;
7332 op->mem_fetch_after = atomic_or_fetch_optab;
7333 op->mem_no_result = atomic_or_optab;
7334 op->fetch_before = sync_old_ior_optab;
7335 op->fetch_after = sync_new_ior_optab;
7336 op->no_result = sync_ior_optab;
7337 op->reverse_code = UNKNOWN;
7338 break;
7339 case NOT:
7340 op->mem_fetch_before = atomic_fetch_nand_optab;
7341 op->mem_fetch_after = atomic_nand_fetch_optab;
7342 op->mem_no_result = atomic_nand_optab;
7343 op->fetch_before = sync_old_nand_optab;
7344 op->fetch_after = sync_new_nand_optab;
7345 op->no_result = sync_nand_optab;
7346 op->reverse_code = UNKNOWN;
7347 break;
7348 default:
7349 gcc_unreachable ();
7350 }
7351 }
7352
7353 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7354 using memory order MODEL. If AFTER is true the operation needs to return
7355 the value of *MEM after the operation, otherwise the previous value.
7356 TARGET is an optional place to place the result. The result is unused if
7357 it is const0_rtx.
7358 Return the result if there is a better sequence, otherwise NULL_RTX. */
7359
7360 static rtx
maybe_optimize_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)7361 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7362 enum memmodel model, bool after)
7363 {
7364 /* If the value is prefetched, or not used, it may be possible to replace
7365 the sequence with a native exchange operation. */
7366 if (!after || target == const0_rtx)
7367 {
7368 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7369 if (code == AND && val == const0_rtx)
7370 {
7371 if (target == const0_rtx)
7372 target = gen_reg_rtx (GET_MODE (mem));
7373 return maybe_emit_atomic_exchange (target, mem, val, model);
7374 }
7375
7376 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7377 if (code == IOR && val == constm1_rtx)
7378 {
7379 if (target == const0_rtx)
7380 target = gen_reg_rtx (GET_MODE (mem));
7381 return maybe_emit_atomic_exchange (target, mem, val, model);
7382 }
7383 }
7384
7385 return NULL_RTX;
7386 }
7387
7388 /* Try to emit an instruction for a specific operation varaition.
7389 OPTAB contains the OP functions.
7390 TARGET is an optional place to return the result. const0_rtx means unused.
7391 MEM is the memory location to operate on.
7392 VAL is the value to use in the operation.
7393 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7394 MODEL is the memory model, if used.
7395 AFTER is true if the returned result is the value after the operation. */
7396
7397 static rtx
maybe_emit_op(const struct atomic_op_functions * optab,rtx target,rtx mem,rtx val,bool use_memmodel,enum memmodel model,bool after)7398 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7399 rtx val, bool use_memmodel, enum memmodel model, bool after)
7400 {
7401 machine_mode mode = GET_MODE (mem);
7402 class expand_operand ops[4];
7403 enum insn_code icode;
7404 int op_counter = 0;
7405 int num_ops;
7406
7407 /* Check to see if there is a result returned. */
7408 if (target == const0_rtx)
7409 {
7410 if (use_memmodel)
7411 {
7412 icode = direct_optab_handler (optab->mem_no_result, mode);
7413 create_integer_operand (&ops[2], model);
7414 num_ops = 3;
7415 }
7416 else
7417 {
7418 icode = direct_optab_handler (optab->no_result, mode);
7419 num_ops = 2;
7420 }
7421 }
7422 /* Otherwise, we need to generate a result. */
7423 else
7424 {
7425 if (use_memmodel)
7426 {
7427 icode = direct_optab_handler (after ? optab->mem_fetch_after
7428 : optab->mem_fetch_before, mode);
7429 create_integer_operand (&ops[3], model);
7430 num_ops = 4;
7431 }
7432 else
7433 {
7434 icode = optab_handler (after ? optab->fetch_after
7435 : optab->fetch_before, mode);
7436 num_ops = 3;
7437 }
7438 create_output_operand (&ops[op_counter++], target, mode);
7439 }
7440 if (icode == CODE_FOR_nothing)
7441 return NULL_RTX;
7442
7443 create_fixed_operand (&ops[op_counter++], mem);
7444 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7445 create_convert_operand_to (&ops[op_counter++], val, mode, true);
7446
7447 if (maybe_expand_insn (icode, num_ops, ops))
7448 return (target == const0_rtx ? const0_rtx : ops[0].value);
7449
7450 return NULL_RTX;
7451 }
7452
7453
7454 /* This function expands an atomic fetch_OP or OP_fetch operation:
7455 TARGET is an option place to stick the return value. const0_rtx indicates
7456 the result is unused.
7457 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7458 CODE is the operation being performed (OP)
7459 MEMMODEL is the memory model variant to use.
7460 AFTER is true to return the result of the operation (OP_fetch).
7461 AFTER is false to return the value before the operation (fetch_OP).
7462
7463 This function will *only* generate instructions if there is a direct
7464 optab. No compare and swap loops or libcalls will be generated. */
7465
7466 static rtx
expand_atomic_fetch_op_no_fallback(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)7467 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7468 enum rtx_code code, enum memmodel model,
7469 bool after)
7470 {
7471 machine_mode mode = GET_MODE (mem);
7472 struct atomic_op_functions optab;
7473 rtx result;
7474 bool unused_result = (target == const0_rtx);
7475
7476 get_atomic_op_for_code (&optab, code);
7477
7478 /* Check to see if there are any better instructions. */
7479 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7480 if (result)
7481 return result;
7482
7483 /* Check for the case where the result isn't used and try those patterns. */
7484 if (unused_result)
7485 {
7486 /* Try the memory model variant first. */
7487 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
7488 if (result)
7489 return result;
7490
7491 /* Next try the old style withuot a memory model. */
7492 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
7493 if (result)
7494 return result;
7495
7496 /* There is no no-result pattern, so try patterns with a result. */
7497 target = NULL_RTX;
7498 }
7499
7500 /* Try the __atomic version. */
7501 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
7502 if (result)
7503 return result;
7504
7505 /* Try the older __sync version. */
7506 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7507 if (result)
7508 return result;
7509
7510 /* If the fetch value can be calculated from the other variation of fetch,
7511 try that operation. */
7512 if (after || unused_result || optab.reverse_code != UNKNOWN)
7513 {
7514 /* Try the __atomic version, then the older __sync version. */
7515 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
7516 if (!result)
7517 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
7518
7519 if (result)
7520 {
7521 /* If the result isn't used, no need to do compensation code. */
7522 if (unused_result)
7523 return result;
7524
7525 /* Issue compensation code. Fetch_after == fetch_before OP val.
7526 Fetch_before == after REVERSE_OP val. */
7527 if (!after)
7528 code = optab.reverse_code;
7529 if (code == NOT)
7530 {
7531 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
7532 true, OPTAB_LIB_WIDEN);
7533 result = expand_simple_unop (mode, NOT, result, target, true);
7534 }
7535 else
7536 result = expand_simple_binop (mode, code, result, val, target,
7537 true, OPTAB_LIB_WIDEN);
7538 return result;
7539 }
7540 }
7541
7542 /* No direct opcode can be generated. */
7543 return NULL_RTX;
7544 }
7545
7546
7547
7548 /* This function expands an atomic fetch_OP or OP_fetch operation:
7549 TARGET is an option place to stick the return value. const0_rtx indicates
7550 the result is unused.
7551 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7552 CODE is the operation being performed (OP)
7553 MEMMODEL is the memory model variant to use.
7554 AFTER is true to return the result of the operation (OP_fetch).
7555 AFTER is false to return the value before the operation (fetch_OP). */
7556 rtx
expand_atomic_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)7557 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7558 enum memmodel model, bool after)
7559 {
7560 machine_mode mode = GET_MODE (mem);
7561 rtx result;
7562 bool unused_result = (target == const0_rtx);
7563
7564 /* If loads are not atomic for the required size and we are not called to
7565 provide a __sync builtin, do not do anything so that we stay consistent
7566 with atomic loads of the same size. */
7567 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7568 return NULL_RTX;
7569
7570 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
7571 after);
7572
7573 if (result)
7574 return result;
7575
7576 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7577 if (code == PLUS || code == MINUS)
7578 {
7579 rtx tmp;
7580 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
7581
7582 start_sequence ();
7583 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
7584 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
7585 model, after);
7586 if (result)
7587 {
7588 /* PLUS worked so emit the insns and return. */
7589 tmp = get_insns ();
7590 end_sequence ();
7591 emit_insn (tmp);
7592 return result;
7593 }
7594
7595 /* PLUS did not work, so throw away the negation code and continue. */
7596 end_sequence ();
7597 }
7598
7599 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7600 if (!can_compare_and_swap_p (mode, false))
7601 {
7602 rtx libfunc;
7603 bool fixup = false;
7604 enum rtx_code orig_code = code;
7605 struct atomic_op_functions optab;
7606
7607 get_atomic_op_for_code (&optab, code);
7608 libfunc = optab_libfunc (after ? optab.fetch_after
7609 : optab.fetch_before, mode);
7610 if (libfunc == NULL
7611 && (after || unused_result || optab.reverse_code != UNKNOWN))
7612 {
7613 fixup = true;
7614 if (!after)
7615 code = optab.reverse_code;
7616 libfunc = optab_libfunc (after ? optab.fetch_before
7617 : optab.fetch_after, mode);
7618 }
7619 if (libfunc != NULL)
7620 {
7621 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7622 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7623 addr, ptr_mode, val, mode);
7624
7625 if (!unused_result && fixup)
7626 result = expand_simple_binop (mode, code, result, val, target,
7627 true, OPTAB_LIB_WIDEN);
7628 return result;
7629 }
7630
7631 /* We need the original code for any further attempts. */
7632 code = orig_code;
7633 }
7634
7635 /* If nothing else has succeeded, default to a compare and swap loop. */
7636 if (can_compare_and_swap_p (mode, true))
7637 {
7638 rtx_insn *insn;
7639 rtx t0 = gen_reg_rtx (mode), t1;
7640
7641 start_sequence ();
7642
7643 /* If the result is used, get a register for it. */
7644 if (!unused_result)
7645 {
7646 if (!target || !register_operand (target, mode))
7647 target = gen_reg_rtx (mode);
7648 /* If fetch_before, copy the value now. */
7649 if (!after)
7650 emit_move_insn (target, t0);
7651 }
7652 else
7653 target = const0_rtx;
7654
7655 t1 = t0;
7656 if (code == NOT)
7657 {
7658 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7659 true, OPTAB_LIB_WIDEN);
7660 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7661 }
7662 else
7663 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7664 OPTAB_LIB_WIDEN);
7665
7666 /* For after, copy the value now. */
7667 if (!unused_result && after)
7668 emit_move_insn (target, t1);
7669 insn = get_insns ();
7670 end_sequence ();
7671
7672 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7673 return target;
7674 }
7675
7676 return NULL_RTX;
7677 }
7678
7679 /* Return true if OPERAND is suitable for operand number OPNO of
7680 instruction ICODE. */
7681
7682 bool
insn_operand_matches(enum insn_code icode,unsigned int opno,rtx operand)7683 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7684 {
7685 return (!insn_data[(int) icode].operand[opno].predicate
7686 || (insn_data[(int) icode].operand[opno].predicate
7687 (operand, insn_data[(int) icode].operand[opno].mode)));
7688 }
7689
7690 /* TARGET is a target of a multiword operation that we are going to
7691 implement as a series of word-mode operations. Return true if
7692 TARGET is suitable for this purpose. */
7693
7694 bool
valid_multiword_target_p(rtx target)7695 valid_multiword_target_p (rtx target)
7696 {
7697 machine_mode mode;
7698 int i, size;
7699
7700 mode = GET_MODE (target);
7701 if (!GET_MODE_SIZE (mode).is_constant (&size))
7702 return false;
7703 for (i = 0; i < size; i += UNITS_PER_WORD)
7704 if (!validate_subreg (word_mode, mode, target, i))
7705 return false;
7706 return true;
7707 }
7708
7709 /* Make OP describe an input operand that has value INTVAL and that has
7710 no inherent mode. This function should only be used for operands that
7711 are always expand-time constants. The backend may request that INTVAL
7712 be copied into a different kind of rtx, but it must specify the mode
7713 of that rtx if so. */
7714
7715 void
create_integer_operand(class expand_operand * op,poly_int64 intval)7716 create_integer_operand (class expand_operand *op, poly_int64 intval)
7717 {
7718 create_expand_operand (op, EXPAND_INTEGER,
7719 gen_int_mode (intval, MAX_MODE_INT),
7720 VOIDmode, false, intval);
7721 }
7722
7723 /* Like maybe_legitimize_operand, but do not change the code of the
7724 current rtx value. */
7725
7726 static bool
maybe_legitimize_operand_same_code(enum insn_code icode,unsigned int opno,class expand_operand * op)7727 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7728 class expand_operand *op)
7729 {
7730 /* See if the operand matches in its current form. */
7731 if (insn_operand_matches (icode, opno, op->value))
7732 return true;
7733
7734 /* If the operand is a memory whose address has no side effects,
7735 try forcing the address into a non-virtual pseudo register.
7736 The check for side effects is important because copy_to_mode_reg
7737 cannot handle things like auto-modified addresses. */
7738 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7739 {
7740 rtx addr, mem;
7741
7742 mem = op->value;
7743 addr = XEXP (mem, 0);
7744 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7745 && !side_effects_p (addr))
7746 {
7747 rtx_insn *last;
7748 machine_mode mode;
7749
7750 last = get_last_insn ();
7751 mode = get_address_mode (mem);
7752 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7753 if (insn_operand_matches (icode, opno, mem))
7754 {
7755 op->value = mem;
7756 return true;
7757 }
7758 delete_insns_since (last);
7759 }
7760 }
7761
7762 return false;
7763 }
7764
7765 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7766 on success, storing the new operand value back in OP. */
7767
7768 static bool
maybe_legitimize_operand(enum insn_code icode,unsigned int opno,class expand_operand * op)7769 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7770 class expand_operand *op)
7771 {
7772 machine_mode mode, imode, tmode;
7773
7774 mode = op->mode;
7775 switch (op->type)
7776 {
7777 case EXPAND_FIXED:
7778 {
7779 temporary_volatile_ok v (true);
7780 return maybe_legitimize_operand_same_code (icode, opno, op);
7781 }
7782
7783 case EXPAND_OUTPUT:
7784 gcc_assert (mode != VOIDmode);
7785 if (op->value
7786 && op->value != const0_rtx
7787 && GET_MODE (op->value) == mode
7788 && maybe_legitimize_operand_same_code (icode, opno, op))
7789 return true;
7790
7791 op->value = gen_reg_rtx (mode);
7792 op->target = 0;
7793 break;
7794
7795 case EXPAND_INPUT:
7796 input:
7797 gcc_assert (mode != VOIDmode);
7798 gcc_assert (GET_MODE (op->value) == VOIDmode
7799 || GET_MODE (op->value) == mode);
7800 if (maybe_legitimize_operand_same_code (icode, opno, op))
7801 return true;
7802
7803 op->value = copy_to_mode_reg (mode, op->value);
7804 break;
7805
7806 case EXPAND_CONVERT_TO:
7807 gcc_assert (mode != VOIDmode);
7808 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
7809 goto input;
7810
7811 case EXPAND_CONVERT_FROM:
7812 if (GET_MODE (op->value) != VOIDmode)
7813 mode = GET_MODE (op->value);
7814 else
7815 /* The caller must tell us what mode this value has. */
7816 gcc_assert (mode != VOIDmode);
7817
7818 imode = insn_data[(int) icode].operand[opno].mode;
7819 tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
7820 ? GET_MODE_INNER (imode) : imode);
7821 if (tmode != VOIDmode && tmode != mode)
7822 {
7823 op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
7824 mode = tmode;
7825 }
7826 if (imode != VOIDmode && imode != mode)
7827 {
7828 gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
7829 op->value = expand_vector_broadcast (imode, op->value);
7830 mode = imode;
7831 }
7832 goto input;
7833
7834 case EXPAND_ADDRESS:
7835 op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
7836 op->value);
7837 goto input;
7838
7839 case EXPAND_INTEGER:
7840 mode = insn_data[(int) icode].operand[opno].mode;
7841 if (mode != VOIDmode
7842 && known_eq (trunc_int_for_mode (op->int_value, mode),
7843 op->int_value))
7844 {
7845 op->value = gen_int_mode (op->int_value, mode);
7846 goto input;
7847 }
7848 break;
7849 }
7850 return insn_operand_matches (icode, opno, op->value);
7851 }
7852
7853 /* Make OP describe an input operand that should have the same value
7854 as VALUE, after any mode conversion that the target might request.
7855 TYPE is the type of VALUE. */
7856
7857 void
create_convert_operand_from_type(class expand_operand * op,rtx value,tree type)7858 create_convert_operand_from_type (class expand_operand *op,
7859 rtx value, tree type)
7860 {
7861 create_convert_operand_from (op, value, TYPE_MODE (type),
7862 TYPE_UNSIGNED (type));
7863 }
7864
7865 /* Return true if the requirements on operands OP1 and OP2 of instruction
7866 ICODE are similar enough for the result of legitimizing OP1 to be
7867 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7868 with OP1 and OP2 respectively. */
7869
7870 static inline bool
can_reuse_operands_p(enum insn_code icode,unsigned int opno1,unsigned int opno2,const class expand_operand * op1,const class expand_operand * op2)7871 can_reuse_operands_p (enum insn_code icode,
7872 unsigned int opno1, unsigned int opno2,
7873 const class expand_operand *op1,
7874 const class expand_operand *op2)
7875 {
7876 /* Check requirements that are common to all types. */
7877 if (op1->type != op2->type
7878 || op1->mode != op2->mode
7879 || (insn_data[(int) icode].operand[opno1].mode
7880 != insn_data[(int) icode].operand[opno2].mode))
7881 return false;
7882
7883 /* Check the requirements for specific types. */
7884 switch (op1->type)
7885 {
7886 case EXPAND_OUTPUT:
7887 /* Outputs must remain distinct. */
7888 return false;
7889
7890 case EXPAND_FIXED:
7891 case EXPAND_INPUT:
7892 case EXPAND_ADDRESS:
7893 case EXPAND_INTEGER:
7894 return true;
7895
7896 case EXPAND_CONVERT_TO:
7897 case EXPAND_CONVERT_FROM:
7898 return op1->unsigned_p == op2->unsigned_p;
7899 }
7900 gcc_unreachable ();
7901 }
7902
7903 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7904 of instruction ICODE. Return true on success, leaving the new operand
7905 values in the OPS themselves. Emit no code on failure. */
7906
7907 bool
maybe_legitimize_operands(enum insn_code icode,unsigned int opno,unsigned int nops,class expand_operand * ops)7908 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
7909 unsigned int nops, class expand_operand *ops)
7910 {
7911 rtx_insn *last = get_last_insn ();
7912 rtx *orig_values = XALLOCAVEC (rtx, nops);
7913 for (unsigned int i = 0; i < nops; i++)
7914 {
7915 orig_values[i] = ops[i].value;
7916
7917 /* First try reusing the result of an earlier legitimization.
7918 This avoids duplicate rtl and ensures that tied operands
7919 remain tied.
7920
7921 This search is linear, but NOPS is bounded at compile time
7922 to a small number (current a single digit). */
7923 unsigned int j = 0;
7924 for (; j < i; ++j)
7925 if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
7926 && rtx_equal_p (orig_values[j], orig_values[i])
7927 && ops[j].value
7928 && insn_operand_matches (icode, opno + i, ops[j].value))
7929 {
7930 ops[i].value = copy_rtx (ops[j].value);
7931 break;
7932 }
7933
7934 /* Otherwise try legitimizing the operand on its own. */
7935 if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
7936 {
7937 delete_insns_since (last);
7938 return false;
7939 }
7940 }
7941 return true;
7942 }
7943
7944 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7945 as its operands. Return the instruction pattern on success,
7946 and emit any necessary set-up code. Return null and emit no
7947 code on failure. */
7948
7949 rtx_insn *
maybe_gen_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7950 maybe_gen_insn (enum insn_code icode, unsigned int nops,
7951 class expand_operand *ops)
7952 {
7953 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
7954 if (!maybe_legitimize_operands (icode, 0, nops, ops))
7955 return NULL;
7956
7957 switch (nops)
7958 {
7959 case 1:
7960 return GEN_FCN (icode) (ops[0].value);
7961 case 2:
7962 return GEN_FCN (icode) (ops[0].value, ops[1].value);
7963 case 3:
7964 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
7965 case 4:
7966 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7967 ops[3].value);
7968 case 5:
7969 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7970 ops[3].value, ops[4].value);
7971 case 6:
7972 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7973 ops[3].value, ops[4].value, ops[5].value);
7974 case 7:
7975 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7976 ops[3].value, ops[4].value, ops[5].value,
7977 ops[6].value);
7978 case 8:
7979 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7980 ops[3].value, ops[4].value, ops[5].value,
7981 ops[6].value, ops[7].value);
7982 case 9:
7983 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7984 ops[3].value, ops[4].value, ops[5].value,
7985 ops[6].value, ops[7].value, ops[8].value);
7986 }
7987 gcc_unreachable ();
7988 }
7989
7990 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7991 as its operands. Return true on success and emit no code on failure. */
7992
7993 bool
maybe_expand_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7994 maybe_expand_insn (enum insn_code icode, unsigned int nops,
7995 class expand_operand *ops)
7996 {
7997 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7998 if (pat)
7999 {
8000 emit_insn (pat);
8001 return true;
8002 }
8003 return false;
8004 }
8005
8006 /* Like maybe_expand_insn, but for jumps. */
8007
8008 bool
maybe_expand_jump_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)8009 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8010 class expand_operand *ops)
8011 {
8012 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8013 if (pat)
8014 {
8015 emit_jump_insn (pat);
8016 return true;
8017 }
8018 return false;
8019 }
8020
8021 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8022 as its operands. */
8023
8024 void
expand_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)8025 expand_insn (enum insn_code icode, unsigned int nops,
8026 class expand_operand *ops)
8027 {
8028 if (!maybe_expand_insn (icode, nops, ops))
8029 gcc_unreachable ();
8030 }
8031
8032 /* Like expand_insn, but for jumps. */
8033
8034 void
expand_jump_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)8035 expand_jump_insn (enum insn_code icode, unsigned int nops,
8036 class expand_operand *ops)
8037 {
8038 if (!maybe_expand_jump_insn (icode, nops, ops))
8039 gcc_unreachable ();
8040 }
8041