;; Machine description for PowerPC synchronization instructions. ;; Copyright (C) 2005-2021 Free Software Foundation, Inc. ;; Contributed by Geoffrey Keating.

;; This file is part of GCC.

;; GCC is free software; you can redistribute it and/or modify it ;; under the terms of the GNU General Public License as published ;; by the Free Software Foundation; either version 3, or (at your ;; option) any later version.

;; GCC is distributed in the hope that it will be useful, but WITHOUT ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public ;; License for more details.

;; You should have received a copy of the GNU General Public License ;; along with GCC; see the file COPYING3. If not see ;; http://www.gnu.org/licenses/.

(define_mode_attr larx [(QI “lbarx”) (HI “lharx”) (SI “lwarx”) (DI “ldarx”) (TI “lqarx”)])

(define_mode_attr stcx [(QI “stbcx.”) (HI “sthcx.”) (SI “stwcx.”) (DI “stdcx.”) (TI “stqcx.”)])

(define_code_iterator FETCHOP [plus minus ior xor and]) (define_code_attr fetchop_name [(plus “add”) (minus “sub”) (ior “or”) (xor “xor”) (and “and”)]) (define_code_attr fetchop_pred [(plus “add_operand”) (minus “int_reg_operand”) (ior “logical_operand”) (xor “logical_operand”) (and “and_operand”)])

(define_expand “mem_thread_fence” [(match_operand:SI 0 “const_int_operand”)] ;; model "" { enum memmodel model = memmodel_base (INTVAL (operands[0])); switch (model) { case MEMMODEL_RELAXED: break; case MEMMODEL_CONSUME: case MEMMODEL_ACQUIRE: case MEMMODEL_RELEASE: case MEMMODEL_ACQ_REL: emit_insn (gen_lwsync ()); break; case MEMMODEL_SEQ_CST: emit_insn (gen_hwsync ()); break; default: gcc_unreachable (); } DONE; })

(define_expand “hwsync” [(set (match_dup 0) (unspec:BLK [(match_dup 0)] UNSPEC_SYNC))] "" { operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); MEM_VOLATILE_P (operands[0]) = 1; })

(define_insn “*hwsync” [(set (match_operand:BLK 0 "" "") (unspec:BLK [(match_dup 0)] UNSPEC_SYNC))] "" “sync” [(set_attr “type” “sync”)])

(define_expand “lwsync” [(set (match_dup 0) (unspec:BLK [(match_dup 0)] UNSPEC_LWSYNC))] "" { operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); MEM_VOLATILE_P (operands[0]) = 1; })

(define_insn “*lwsync” [(set (match_operand:BLK 0 "" "") (unspec:BLK [(match_dup 0)] UNSPEC_LWSYNC))] "" { if (TARGET_NO_LWSYNC) return “sync”; else return “lwsync”; } [(set_attr “type” “sync”)])

(define_insn “isync” [(unspec_volatile:BLK [(const_int 0)] UNSPECV_ISYNC)] "" “isync” [(set_attr “type” “isync”)])

;; Types that we should provide atomic instructions for. (define_mode_iterator AINT [QI HI SI (DI “TARGET_POWERPC64”) (TI “TARGET_SYNC_TI”)])

;; The control dependency used for load dependency described ;; in B.2.3 of the Power ISA 2.06B. (define_insn “loadsync_” [(unspec_volatile:BLK [(match_operand:AINT 0 “register_operand” “r”)] UNSPECV_ISYNC) (clobber (match_scratch:CC 1 “=y”))] "" “cmpw %1,%0,%0;bne- %1,$+4;isync” [(set_attr “type” “isync”) (set_attr “length” “12”)])

;; If TARGET_PREFIXED, always use plq rather than lq. (define_insn “load_quadpti” [(set (match_operand:PTI 0 “quad_int_reg_operand” “=&r”) (unspec:PTI [(match_operand:TI 1 “quad_memory_operand” “wQ”)] UNSPEC_LSQ))] “TARGET_SYNC_TI && !reg_mentioned_p (operands[0], operands[1])” “lq %0,%1” [(set_attr “type” “load”) (set_attr “size” “128”) (set (attr “prefixed”) (if_then_else (match_test “TARGET_PREFIXED”) (const_string “yes”) (const_string “no”)))])

;; Pattern load_quadpti will always use plq for atomic TImode if ;; TARGET_PREFIXED. It has the correct doubleword ordering on either LE ;; or BE, so we can just move the result into the output register and ;; do not need to do the doubleword swap for LE. Also this avoids any ;; confusion about whether the lq vs plq might be used based on whether ;; op1 has PC-relative addressing. We could potentially allow BE to ;; use lq because it doesn't have the doubleword ordering problem. (define_expand “atomic_load” [(set (match_operand:AINT 0 “register_operand”) ;; output (match_operand:AINT 1 “memory_operand”)) ;; memory (use (match_operand:SI 2 “const_int_operand”))] ;; model "" { if (mode == TImode && !TARGET_SYNC_TI) FAIL;

enum memmodel model = memmodel_base (INTVAL (operands[2]));

if (is_mm_seq_cst (model)) emit_insn (gen_hwsync ());

if (mode != TImode) emit_move_insn (operands[0], operands[1]); else { rtx op0 = operands[0]; rtx op1 = operands[1]; rtx pti_reg = gen_reg_rtx (PTImode);

  if (!quad_address_p (XEXP (op1, 0), TImode, false))
{
  rtx old_addr = XEXP (op1, 0);
  rtx new_addr = force_reg (Pmode, old_addr);
  operands[1] = op1 = replace_equiv_address (op1, new_addr);
}

  emit_insn (gen_load_quadpti (pti_reg, op1));

  if (WORDS_BIG_ENDIAN || TARGET_PREFIXED)
emit_move_insn (op0, gen_lowpart (TImode, pti_reg));
  else
{
  emit_move_insn (gen_lowpart (DImode, op0), gen_highpart (DImode, pti_reg));
  emit_move_insn (gen_highpart (DImode, op0), gen_lowpart (DImode, pti_reg));
}
}

switch (model) { case MEMMODEL_RELAXED: break; case MEMMODEL_CONSUME: case MEMMODEL_ACQUIRE: case MEMMODEL_SEQ_CST: emit_insn (gen_loadsync_ (operands[0])); break; default: gcc_unreachable (); } DONE; })

;; If TARGET_PREFIXED, always use pstq rather than stq. (define_insn “store_quadpti” [(set (match_operand:PTI 0 “quad_memory_operand” “=wQ”) (unspec:PTI [(match_operand:PTI 1 “quad_int_reg_operand” “r”)] UNSPEC_LSQ))] “TARGET_SYNC_TI” “stq %1,%0” [(set_attr “type” “store”) (set_attr “size” “128”) (set (attr “prefixed”) (if_then_else (match_test “TARGET_PREFIXED”) (const_string “yes”) (const_string “no”)))])

;; Pattern store_quadpti will always use pstq if TARGET_PREFIXED, ;; so the doubleword swap is never needed in that case. (define_expand “atomic_store” [(set (match_operand:AINT 0 “memory_operand”) ;; memory (match_operand:AINT 1 “register_operand”)) ;; input (use (match_operand:SI 2 “const_int_operand”))] ;; model "" { if (mode == TImode && !TARGET_SYNC_TI) FAIL;

enum memmodel model = memmodel_base (INTVAL (operands[2])); switch (model) { case MEMMODEL_RELAXED: break; case MEMMODEL_RELEASE: emit_insn (gen_lwsync ()); break; case MEMMODEL_SEQ_CST: emit_insn (gen_hwsync ()); break; default: gcc_unreachable (); } if (mode != TImode) emit_move_insn (operands[0], operands[1]); else { rtx op0 = operands[0]; rtx op1 = operands[1]; rtx pti_reg = gen_reg_rtx (PTImode);

  if (!quad_address_p (XEXP (op0, 0), TImode, false))
{
  rtx old_addr = XEXP (op0, 0);
  rtx new_addr = force_reg (Pmode, old_addr);
  operands[0] = op0 = replace_equiv_address (op0, new_addr);
}

  if (WORDS_BIG_ENDIAN || TARGET_PREFIXED)
emit_move_insn (pti_reg, gen_lowpart (PTImode, op1));
  else
{
  emit_move_insn (gen_lowpart (DImode, pti_reg), gen_highpart (DImode, op1));
  emit_move_insn (gen_highpart (DImode, pti_reg), gen_lowpart (DImode, op1));
}

  emit_insn (gen_store_quadpti (gen_lowpart (PTImode, op0), pti_reg));
}

DONE; })

;; Any supported integer mode that has atomic larx/stcx. instrucitons ;; other than the quad memory operations, which have special restrictions. ;; Byte/halfword atomic instructions were added in ISA 2.06B, but were phased ;; in and did not show up until power8. TImode atomic lqarx/stqcx. require ;; special handling due to even/odd register requirements. (define_mode_iterator ATOMIC [(QI “TARGET_SYNC_HI_QI”) (HI “TARGET_SYNC_HI_QI”) SI (DI “TARGET_POWERPC64”)])

(define_insn “load_locked” [(set (match_operand:ATOMIC 0 “int_reg_operand” “=r”) (unspec_volatile:ATOMIC [(match_operand:ATOMIC 1 “memory_operand” “Z”)] UNSPECV_LL))] "" “ %0,%y1” [(set_attr “type” “load_l”)])

(define_insn “load_lockedQHI:mode_si” [(set (match_operand:SI 0 “int_reg_operand” “=r”) (unspec_volatile:SI [(match_operand:QHI 1 “memory_operand” “Z”)] UNSPECV_LL))] “TARGET_SYNC_HI_QI” “QHI:larx %0,%y1” [(set_attr “type” “load_l”)])

;; Use PTImode to get even/odd register pairs. ;; Use a temporary register to force getting an even register for the ;; lqarx/stqcrx. instructions. Normal optimizations will eliminate this extra ;; copy on big endian systems.

;; On little endian systems where non-atomic quad word load/store instructions ;; are not used, the address can be register+offset, so make sure the address ;; is indexed or indirect before register allocation.

(define_expand “load_lockedti” [(use (match_operand:TI 0 “quad_int_reg_operand”)) (use (match_operand:TI 1 “memory_operand”))] “TARGET_SYNC_TI” { rtx op0 = operands[0]; rtx op1 = operands[1]; rtx pti = gen_reg_rtx (PTImode);

if (!indexed_or_indirect_operand (op1, TImode)) { rtx old_addr = XEXP (op1, 0); rtx new_addr = force_reg (Pmode, old_addr); operands[1] = op1 = change_address (op1, TImode, new_addr); }

emit_insn (gen_load_lockedpti (pti, op1)); if (WORDS_BIG_ENDIAN) emit_move_insn (op0, gen_lowpart (TImode, pti)); else { emit_move_insn (gen_lowpart (DImode, op0), gen_highpart (DImode, pti)); emit_move_insn (gen_highpart (DImode, op0), gen_lowpart (DImode, pti)); } DONE; })

(define_insn “load_lockedpti” [(set (match_operand:PTI 0 “quad_int_reg_operand” “=&r”) (unspec_volatile:PTI [(match_operand:TI 1 “indexed_or_indirect_operand” “Z”)] UNSPECV_LL))] “TARGET_SYNC_TI && !reg_mentioned_p (operands[0], operands[1]) && quad_int_reg_operand (operands[0], PTImode)” “lqarx %0,%y1” [(set_attr “type” “load_l”) (set_attr “size” “128”)])

(define_insn “store_conditional” [(set (match_operand:CC 0 “cc_reg_operand” “=x”) (unspec_volatile:CC [(const_int 0)] UNSPECV_SC)) (set (match_operand:ATOMIC 1 “memory_operand” “=Z”) (match_operand:ATOMIC 2 “int_reg_operand” “r”))] "" “ %2,%y1” [(set_attr “type” “store_c”)])

;; Use a temporary register to force getting an even register for the ;; lqarx/stqcrx. instructions. Normal optimizations will eliminate this extra ;; copy on big endian systems.

;; On little endian systems where non-atomic quad word load/store instructions ;; are not used, the address can be register+offset, so make sure the address ;; is indexed or indirect before register allocation.

(define_expand “store_conditionalti” [(use (match_operand:CC 0 “cc_reg_operand”)) (use (match_operand:TI 1 “memory_operand”)) (use (match_operand:TI 2 “quad_int_reg_operand”))] “TARGET_SYNC_TI” { rtx op0 = operands[0]; rtx op1 = operands[1]; rtx op2 = operands[2]; rtx addr = XEXP (op1, 0); rtx pti_mem; rtx pti_reg;

if (!indexed_or_indirect_operand (op1, TImode)) { rtx new_addr = force_reg (Pmode, addr); operands[1] = op1 = change_address (op1, TImode, new_addr); addr = new_addr; }

pti_mem = change_address (op1, PTImode, addr); pti_reg = gen_reg_rtx (PTImode);

if (WORDS_BIG_ENDIAN) emit_move_insn (pti_reg, gen_lowpart (PTImode, op2)); else { emit_move_insn (gen_lowpart (DImode, pti_reg), gen_highpart (DImode, op2)); emit_move_insn (gen_highpart (DImode, pti_reg), gen_lowpart (DImode, op2)); }

emit_insn (gen_store_conditionalpti (op0, pti_mem, pti_reg)); DONE; })

(define_insn “store_conditionalpti” [(set (match_operand:CC 0 “cc_reg_operand” “=x”) (unspec_volatile:CC [(const_int 0)] UNSPECV_SC)) (set (match_operand:PTI 1 “indexed_or_indirect_operand” “=Z”) (match_operand:PTI 2 “quad_int_reg_operand” “r”))] “TARGET_SYNC_TI && quad_int_reg_operand (operands[2], PTImode)” “stqcx. %2,%y1” [(set_attr “type” “store_c”) (set_attr “size” “128”)])

(define_expand “atomic_compare_and_swap” [(match_operand:SI 0 “int_reg_operand”) ;; bool out (match_operand:AINT 1 “int_reg_operand”) ;; val out (match_operand:AINT 2 “memory_operand”) ;; memory (match_operand:AINT 3 “reg_or_short_operand”) ;; expected (match_operand:AINT 4 “int_reg_operand”) ;; desired (match_operand:SI 5 “const_int_operand”) ;; is_weak (match_operand:SI 6 “const_int_operand”) ;; model succ (match_operand:SI 7 “const_int_operand”)] ;; model fail "" { rs6000_expand_atomic_compare_and_swap (operands); DONE; })

(define_expand “atomic_exchange” [(match_operand:AINT 0 “int_reg_operand”) ;; output (match_operand:AINT 1 “memory_operand”) ;; memory (match_operand:AINT 2 “int_reg_operand”) ;; input (match_operand:SI 3 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_exchange (operands); DONE; })

(define_expand “atomic_<fetchop_name>” [(match_operand:AINT 0 “memory_operand”) ;; memory (FETCHOP:AINT (match_dup 0) (match_operand:AINT 1 “<fetchop_pred>”)) ;; operand (match_operand:SI 2 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_op (, operands[0], operands[1], NULL_RTX, NULL_RTX, operands[2]); DONE; })

(define_expand “atomic_nand” [(match_operand:AINT 0 “memory_operand”) ;; memory (match_operand:AINT 1 “int_reg_operand”) ;; operand (match_operand:SI 2 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_op (NOT, operands[0], operands[1], NULL_RTX, NULL_RTX, operands[2]); DONE; })

(define_expand “atomic_fetch_<fetchop_name>” [(match_operand:AINT 0 “int_reg_operand”) ;; output (match_operand:AINT 1 “memory_operand”) ;; memory (FETCHOP:AINT (match_dup 1) (match_operand:AINT 2 “<fetchop_pred>”)) ;; operand (match_operand:SI 3 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_op (, operands[1], operands[2], operands[0], NULL_RTX, operands[3]); DONE; })

(define_expand “atomic_fetch_nand” [(match_operand:AINT 0 “int_reg_operand”) ;; output (match_operand:AINT 1 “memory_operand”) ;; memory (match_operand:AINT 2 “int_reg_operand”) ;; operand (match_operand:SI 3 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_op (NOT, operands[1], operands[2], operands[0], NULL_RTX, operands[3]); DONE; })

(define_expand “atomic_<fetchop_name>_fetch” [(match_operand:AINT 0 “int_reg_operand”) ;; output (match_operand:AINT 1 “memory_operand”) ;; memory (FETCHOP:AINT (match_dup 1) (match_operand:AINT 2 “<fetchop_pred>”)) ;; operand (match_operand:SI 3 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_op (, operands[1], operands[2], NULL_RTX, operands[0], operands[3]); DONE; })

(define_expand “atomic_nand_fetch” [(match_operand:AINT 0 “int_reg_operand”) ;; output (match_operand:AINT 1 “memory_operand”) ;; memory (match_operand:AINT 2 “int_reg_operand”) ;; operand (match_operand:SI 3 “const_int_operand”)] ;; model "" { rs6000_expand_atomic_op (NOT, operands[1], operands[2], NULL_RTX, operands[0], operands[3]); DONE; })