AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950 #116678

arsenm · 2024-11-18T19:01:44Z

No description provided.

arsenm · 2024-11-18T19:02:01Z

This stack of pull requests is managed by Graphite. Learn more about stacking.

llvmbot · 2024-11-18T19:03:32Z

@llvm/pr-subscribers-llvm-ir
@llvm/pr-subscribers-mc

@llvm/pr-subscribers-backend-amdgpu

Author: Matt Arsenault (arsenm)

Changes

Patch is 27.17 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/116678.diff

6 Files Affected:

(modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+6)
(modified) llvm/lib/Target/AMDGPU/SIInstrInfo.td (+1)
(modified) llvm/lib/Target/AMDGPU/VOP3Instructions.td (+25)
(modified) llvm/test/CodeGen/AMDGPU/bf16-conversions.ll (+178-217)
(added) llvm/test/MC/AMDGPU/gfx950_asm_vop3.s (+26)
(added) llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt (+19)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1e261f4256c93b..ad89812558d25c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -889,6 +889,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::MUL, MVT::i1, Promote);
 
+  if (Subtarget->hasBF16ConversionInsts()) {
+    setOperationAction(ISD::FP_ROUND, MVT::v2bf16, Legal);
+    setOperationAction(ISD::FP_ROUND, MVT::bf16, Legal);
+    setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
+  }
+
   setTargetDAGCombine({ISD::ADD,
                        ISD::UADDO_CARRY,
                        ISD::SUB,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 882e147dc231fa..7df9be5c6f7a0b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -2787,6 +2787,7 @@ def VOP_I32_I32_I32 : VOPProfile <[i32, i32, i32, untyped]>;
 def VOP_I32_I32_I32_ARITH : VOPProfile <[i32, i32, i32, untyped], /*EnableClamp=*/1>;
 def VOP_V2F16_F32_F32 : VOPProfile <[v2f16, f32, f32, untyped]>;
 def VOP_F32_F16_F16_F16 : VOPProfile <[f32, f16, f16, f16]>;
+def VOP_V2BF16_F32_F32 : VOPProfile <[v2bf16, f32, f32, untyped]>;
 
 def VOP_I64_I64_I32 : VOPProfile <[i64, i64, i32, untyped]>;
 def VOP_I64_I32_I64 : VOPProfile <[i64, i32, i64, untyped]>;
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 551e8b3a679202..917e1b3974b46a 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -944,6 +944,30 @@ let SubtargetPredicate = isGFX11Plus in {
   defm V_CVT_PK_U16_F32 : VOP3Inst<"v_cvt_pk_u16_f32", VOP3_Profile<VOP_V2I16_F32_F32>>;
 } // End SubtargetPredicate = isGFX11Plus
 
+// FIXME: GlobalISel cannot distinguish f16 and bf16 and may start using bf16 patterns
+//        instead of less complex f16. Disable GlobalISel for these for now.
+def bf16_fpround : PatFrag <(ops node:$src0),  (fpround $src0), [{ return true; }]> {
+  let GISelPredicateCode = [{return false;}];
+}
+
+let SubtargetPredicate = HasBF16ConversionInsts in {
+  let ReadsModeReg = 0 in {
+    defm V_CVT_PK_BF16_F32    : VOP3Inst<"v_cvt_pk_bf16_f32", VOP3_Profile<VOP_V2BF16_F32_F32>>;
+  }
+  def : GCNPat<(v2bf16 (bf16_fpround v2f32:$src)),
+               (V_CVT_PK_BF16_F32_e64 0, (EXTRACT_SUBREG VReg_64:$src, sub0), 0, (EXTRACT_SUBREG VReg_64:$src, sub1))>;
+  def : GCNPat<(v2bf16 (bf16_fpround v2f64:$src)),
+               (V_CVT_PK_BF16_F32_e64 0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub0_sub1)),
+                                      0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub2_sub3)))>;
+  def : GCNPat<(v2bf16 (build_vector (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),
+                                     (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)))))),
+               (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1)>;
+  def : GCNPat<(bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),
+               (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, 0, (f32 (IMPLICIT_DEF)))>;
+  def : GCNPat<(bf16 (bf16_fpround (f64 (VOP3Mods f64:$src0, i32:$src0_modifiers)))),
+               (V_CVT_PK_BF16_F32_e64 0, (f32 (V_CVT_F32_F64_e64 $src0_modifiers, $src0)), 0, (f32 (IMPLICIT_DEF)))>;
+}
+
 let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
   defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
   defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
@@ -1721,5 +1745,6 @@ defm V_LSHL_ADD_U64 : VOP3_Real_vi <0x208>;
 
 defm V_CVT_PK_FP8_F32 : VOP3OpSel_Real_gfx9 <0x2a2>;
 defm V_CVT_PK_BF8_F32 : VOP3OpSel_Real_gfx9 <0x2a3>;
+defm V_CVT_PK_BF16_F32: VOP3OpSel_Real_gfx9 <0x268>;
 defm V_CVT_SR_FP8_F32 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x2a4>;
 defm V_CVT_SR_BF8_F32 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x2a5>;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 425fc5884cec7f..135efceb31fdda 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -24,139 +24,168 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
 }
 
 define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_v(<2 x float> %src) {
-; GCN-LABEL: v_test_cvt_v2f32_v2bf16_v:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GCN-NEXT:    s_movk_i32 s0, 0x7fff
-; GCN-NEXT:    v_add3_u32 v2, v2, v0, s0
-; GCN-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GCN-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GCN-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GCN-NEXT:    v_add3_u32 v2, v2, v1, s0
-; GCN-NEXT:    v_or_b32_e32 v3, 0x400000, v1
-; GCN-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GCN-NEXT:    s_mov_b32 s0, 0x7060302
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
-; GCN-NEXT:    v_perm_b32 v0, v1, v0, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX-940-LABEL: v_test_cvt_v2f32_v2bf16_v:
+; GFX-940:       ; %bb.0:
+; GFX-940-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX-940-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX-940-NEXT:    v_add3_u32 v2, v2, v0, s0
+; GFX-940-NEXT:    v_or_b32_e32 v3, 0x400000, v0
+; GFX-940-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX-940-NEXT:    s_nop 1
+; GFX-940-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX-940-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX-940-NEXT:    v_add3_u32 v2, v2, v1, s0
+; GFX-940-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX-940-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX-940-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX-940-NEXT:    s_nop 0
+; GFX-940-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX-940-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX-940-NEXT:    ; return to shader part epilog
+;
+; GFX-950-LABEL: v_test_cvt_v2f32_v2bf16_v:
+; GFX-950:       ; %bb.0:
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v1
+; GFX-950-NEXT:    ; return to shader part epilog
   %res = fptrunc <2 x float> %src to <2 x bfloat>
   %cast = bitcast <2 x bfloat> %res to float
   ret float %cast
 }
 
 define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {
-; GCN-LABEL: v_test_cvt_v2f32_v2bf16_s:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_bfe_u32 s2, s1, 0x10010
-; GCN-NEXT:    s_add_i32 s2, s2, s1
-; GCN-NEXT:    s_or_b32 s4, s1, 0x400000
-; GCN-NEXT:    s_add_i32 s5, s2, 0x7fff
-; GCN-NEXT:    v_cmp_u_f32_e64 s[2:3], s1, s1
-; GCN-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GCN-NEXT:    s_cselect_b32 s1, s4, s5
-; GCN-NEXT:    s_lshr_b32 s2, s1, 16
-; GCN-NEXT:    s_bfe_u32 s1, s0, 0x10010
-; GCN-NEXT:    s_add_i32 s1, s1, s0
-; GCN-NEXT:    s_or_b32 s3, s0, 0x400000
-; GCN-NEXT:    s_add_i32 s4, s1, 0x7fff
-; GCN-NEXT:    v_cmp_u_f32_e64 s[0:1], s0, s0
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT:    s_cselect_b32 s0, s3, s4
-; GCN-NEXT:    s_lshr_b32 s0, s0, 16
-; GCN-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX-940-LABEL: v_test_cvt_v2f32_v2bf16_s:
+; GFX-940:       ; %bb.0:
+; GFX-940-NEXT:    s_bfe_u32 s2, s1, 0x10010
+; GFX-940-NEXT:    s_add_i32 s2, s2, s1
+; GFX-940-NEXT:    s_or_b32 s4, s1, 0x400000
+; GFX-940-NEXT:    s_add_i32 s5, s2, 0x7fff
+; GFX-940-NEXT:    v_cmp_u_f32_e64 s[2:3], s1, s1
+; GFX-940-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; GFX-940-NEXT:    s_cselect_b32 s1, s4, s5
+; GFX-940-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX-940-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX-940-NEXT:    s_add_i32 s1, s1, s0
+; GFX-940-NEXT:    s_or_b32 s3, s0, 0x400000
+; GFX-940-NEXT:    s_add_i32 s4, s1, 0x7fff
+; GFX-940-NEXT:    v_cmp_u_f32_e64 s[0:1], s0, s0
+; GFX-940-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX-940-NEXT:    s_cselect_b32 s0, s3, s4
+; GFX-940-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX-940-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX-940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX-940-NEXT:    ; return to shader part epilog
+;
+; GFX-950-LABEL: v_test_cvt_v2f32_v2bf16_s:
+; GFX-950:       ; %bb.0:
+; GFX-950-NEXT:    v_mov_b32_e32 v0, s1
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, s0, v0
+; GFX-950-NEXT:    ; return to shader part epilog
   %res = fptrunc <2 x float> %src to <2 x bfloat>
   %cast = bitcast <2 x bfloat> %res to float
   ret float %cast
 }
 
 define amdgpu_ps float @v_test_cvt_f32_bf16_v(float %src) {
-; GCN-LABEL: v_test_cvt_f32_bf16_v:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GCN-NEXT:    s_movk_i32 s0, 0x7fff
-; GCN-NEXT:    v_add3_u32 v1, v1, v0, s0
-; GCN-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GCN-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX-940-LABEL: v_test_cvt_f32_bf16_v:
+; GFX-940:       ; %bb.0:
+; GFX-940-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX-940-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX-940-NEXT:    v_add3_u32 v1, v1, v0, s0
+; GFX-940-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX-940-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX-940-NEXT:    s_nop 1
+; GFX-940-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX-940-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX-940-NEXT:    ; return to shader part epilog
+;
+; GFX-950-LABEL: v_test_cvt_f32_bf16_v:
+; GFX-950:       ; %bb.0:
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX-950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX-950-NEXT:    ; return to shader part epilog
   %trunc = fptrunc float %src to bfloat
   %ext = fpext bfloat %trunc to float
   ret float %ext
 }
 
 define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) {
-; GCN-LABEL: v_test_cvt_v2f64_v2bf16_v:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_cvt_f32_f64_e64 v6, |v[0:1]|
-; GCN-NEXT:    v_cvt_f64_f32_e32 v[4:5], v6
-; GCN-NEXT:    v_and_b32_e32 v7, 1, v6
-; GCN-NEXT:    v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5]
-; GCN-NEXT:    v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5]
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v4, -1, 1, s[2:3]
-; GCN-NEXT:    v_add_u32_e32 v4, v6, v4
-; GCN-NEXT:    s_or_b64 vcc, s[0:1], vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GCN-NEXT:    s_brev_b32 s4, 1
-; GCN-NEXT:    v_and_or_b32 v5, v1, s4, v4
-; GCN-NEXT:    v_bfe_u32 v4, v4, 16, 1
-; GCN-NEXT:    s_movk_i32 s5, 0x7fff
-; GCN-NEXT:    v_add3_u32 v4, v4, v5, s5
-; GCN-NEXT:    v_or_b32_e32 v5, 0x400000, v5
-; GCN-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[0:1]
-; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GCN-NEXT:    v_cvt_f32_f64_e64 v5, |v[2:3]|
-; GCN-NEXT:    v_cvt_f64_f32_e32 v[0:1], v5
-; GCN-NEXT:    v_and_b32_e32 v6, 1, v5
-; GCN-NEXT:    v_cmp_gt_f64_e64 s[2:3], |v[2:3]|, v[0:1]
-; GCN-NEXT:    v_cmp_nlg_f64_e64 s[0:1], |v[2:3]|, v[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v0, -1, 1, s[2:3]
-; GCN-NEXT:    v_add_u32_e32 v0, v5, v0
-; GCN-NEXT:    s_or_b64 vcc, s[0:1], vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
-; GCN-NEXT:    v_and_or_b32 v1, v3, s4, v0
-; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 1
-; GCN-NEXT:    v_add3_u32 v0, v0, v1, s5
-; GCN-NEXT:    v_or_b32_e32 v1, 0x400000, v1
-; GCN-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[2:3]
-; GCN-NEXT:    s_mov_b32 s0, 0x7060302
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_perm_b32 v0, v0, v4, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX-940-LABEL: v_test_cvt_v2f64_v2bf16_v:
+; GFX-940:       ; %bb.0:
+; GFX-940-NEXT:    v_cvt_f32_f64_e64 v6, |v[0:1]|
+; GFX-940-NEXT:    v_cvt_f64_f32_e32 v[4:5], v6
+; GFX-940-NEXT:    v_and_b32_e32 v7, 1, v6
+; GFX-940-NEXT:    v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5]
+; GFX-940-NEXT:    v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5]
+; GFX-940-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v7
+; GFX-940-NEXT:    v_cndmask_b32_e64 v4, -1, 1, s[2:3]
+; GFX-940-NEXT:    v_add_u32_e32 v4, v6, v4
+; GFX-940-NEXT:    s_or_b64 vcc, s[0:1], vcc
+; GFX-940-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX-940-NEXT:    s_brev_b32 s4, 1
+; GFX-940-NEXT:    v_and_or_b32 v5, v1, s4, v4
+; GFX-940-NEXT:    v_bfe_u32 v4, v4, 16, 1
+; GFX-940-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX-940-NEXT:    v_add3_u32 v4, v4, v5, s5
+; GFX-940-NEXT:    v_or_b32_e32 v5, 0x400000, v5
+; GFX-940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[0:1]
+; GFX-940-NEXT:    s_nop 1
+; GFX-940-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX-940-NEXT:    v_cvt_f32_f64_e64 v5, |v[2:3]|
+; GFX-940-NEXT:    v_cvt_f64_f32_e32 v[0:1], v5
+; GFX-940-NEXT:    v_and_b32_e32 v6, 1, v5
+; GFX-940-NEXT:    v_cmp_gt_f64_e64 s[2:3], |v[2:3]|, v[0:1]
+; GFX-940-NEXT:    v_cmp_nlg_f64_e64 s[0:1], |v[2:3]|, v[0:1]
+; GFX-940-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX-940-NEXT:    v_cndmask_b32_e64 v0, -1, 1, s[2:3]
+; GFX-940-NEXT:    v_add_u32_e32 v0, v5, v0
+; GFX-940-NEXT:    s_or_b64 vcc, s[0:1], vcc
+; GFX-940-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX-940-NEXT:    v_and_or_b32 v1, v3, s4, v0
+; GFX-940-NEXT:    v_bfe_u32 v0, v0, 16, 1
+; GFX-940-NEXT:    v_add3_u32 v0, v0, v1, s5
+; GFX-940-NEXT:    v_or_b32_e32 v1, 0x400000, v1
+; GFX-940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[2:3]
+; GFX-940-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX-940-NEXT:    s_nop 0
+; GFX-940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX-940-NEXT:    v_perm_b32 v0, v0, v4, s0
+; GFX-940-NEXT:    ; return to shader part epilog
+;
+; GFX-950-LABEL: v_test_cvt_v2f64_v2bf16_v:
+; GFX-950:       ; %bb.0:
+; GFX-950-NEXT:    v_cvt_f32_f64_e32 v2, v[2:3]
+; GFX-950-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v2
+; GFX-950-NEXT:    ; return to shader part epilog
   %res = fptrunc <2 x double> %src to <2 x bfloat>
   %cast = bitcast <2 x bfloat> %res to float
   ret float %cast
 }
 
 define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16(float %a, float %b) {
-; GCN-LABEL: fptrunc_f32_f32_to_v2bf16:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GCN-NEXT:    s_movk_i32 s0, 0x7fff
-; GCN-NEXT:    v_add3_u32 v2, v2, v0, s0
-; GCN-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GCN-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GCN-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GCN-NEXT:    v_add3_u32 v2, v2, v1, s0
-; GCN-NEXT:    v_or_b32_e32 v3, 0x400000, v1
-; GCN-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GCN-NEXT:    s_mov_b32 s0, 0x7060302
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
-; GCN-NEXT:    v_perm_b32 v0, v1, v0, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX-940-LABEL: fptrunc_f32_f32_to_v2bf16:
+; GFX-940:       ; %bb.0: ; %entry
+; GFX-940-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX-940-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX-940-NEXT:    v_add3_u32 v2, v2, v0, s0
+; GFX-940-NEXT:    v_or_b32_e32 v3, 0x400000, v0
+; GFX-940-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX-940-NEXT:    s_nop 1
+; GFX-940-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX-940-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX-940-NEXT:    v_add3_u32 v2, v2, v1, s0
+; GFX-940-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX-940-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX-940-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX-940-NEXT:    s_nop 0
+; GFX-940-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX-940-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX-940-NEXT:    ; return to shader part epilog
+;
+; GFX-950-LABEL: fptrunc_f32_f32_to_v2bf16:
+; GFX-950:       ; %bb.0: ; %entry
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v1
+; GFX-950-NEXT:    ; return to shader part epilog
 entry:
   %a.cvt = fptrunc float %a to bfloat
   %b.cvt = fptrunc float %b to bfloat
@@ -167,26 +196,31 @@ entry:
 }
 
 define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16_mods(float %a, float %b) {
-; GCN-LABEL: fptrunc_f32_f32_to_v2bf16_mods:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    v_xor_b32_e32 v2, 0x80000000, v0
-; GCN-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GCN-NEXT:    s_movk_i32 s0, 0x7fff
-; GCN-NEXT:    v_add3_u32 v3, v3, v2, s0
-; GCN-NEXT:    v_or_b32_e32 v2, 0x400000, v2
-; GCN-NEXT:    v_cmp_u_f32_e64 vcc, -v0, -v0
-; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
-; GCN-NEXT:    v_and_b32_e32 v2, 0x7fffffff, v1
-; GCN-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GCN-NEXT:    v_add3_u32 v3, v3, v2, s0
-; GCN-NEXT:    v_or_b32_e32 v2, 0x400000, v2
-; GCN-NEXT:    v_cmp_u_f32_e64 vcc, |v1|, |v1|
-; GCN-NEXT:    s_mov_b32 s0, 0x7060302
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
-; GCN-NEXT:    v_perm_b32 v0, v1, v0, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX-940-LABEL: fptrunc_f32_f32_to_v2bf16_mods:
+; GFX-940:       ; %bb.0: ; %entry
+; GFX-940-NEXT:    v_xor_b32_e32 v2, 0x80000000, v0
+; GFX-940-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX-940-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX-940-NEXT:    v_add3_u32 v3, v3, v2, s0
+; GFX-940-NEXT:    v_or_b32_e32 v2, 0x400000, v2
+; GFX-940-NEXT:    v_cmp_u_f32_e64 vcc, -v0, -v0
+; GFX-940-NEXT:    s_nop 1
+; GFX-940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX-940-NEXT:    v_and_b32_e32 v2, 0x7fffffff, v1
+; GFX-940-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX-940-NEXT:    v_add3_u32 v3, v3, v2, s0
+; GFX-940-NEXT:    v_or_b32_e32 v2, 0x400000, v2
+; GFX-940-NEXT:    v_cmp_u_f32_e64 vcc, |v1|, |v1|
+; GFX-940-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX-940-NEXT:    s_nop 0
+; GFX-940-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX-940-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX-940-NEXT:    ; return to shader part epilog
+;
+; GFX-950-LABEL: fptrunc_f32_f32_to_v2bf16_mods:
+; GFX-950:       ; %bb.0: ; %entry
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, -v0, |v1|
+; GFX-950-NEXT:    ; return to shader part epilog
 entry:
   %a.neg = fneg float %a
   %a.cvt = fptrunc float %a.neg to bfloat
@@ -217,14 +251,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16(float %a, ptr %out) {
 ; GFX-950:       ; %bb.0: ; %entry
 ; GFX-950-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX-950-NEXT:    v_mov_b32_e32 v2, v1
-; GFX-950-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX-950-NEXT:    s_movk_i32 s0, 0x7fff
-; GFX-950-NEXT:    v_add3_u32 v1, v1, v0, s0
-; GFX-950-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX-950-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX-950-NEXT:    s_nop 1
-; GFX-950-NEXT:    v_cndmask_b32_e32 v0, v1, v4, vcc
-; GFX-950-NEXT:    flat_store_short_d16_hi v[2:3], v0
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX-950-NEXT:    flat_store_short v[2:3], v0
 ; GFX-950-NEXT:    s_endpgm
 entry:
   %a.cvt = fptrunc float %a to bfloat
@@ -252,15 +280,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_abs(float %a, ptr %out) {
 ; GFX-950:       ; %bb.0: ; %entry
 ; GFX-950-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX-950-NEXT:    v_mov_b32_e32 v2, v1
-; GFX-950-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v0
-; GFX-950-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX-950-NEXT:    s_movk_i32 s0, 0x7fff
-; GFX-950-NEXT:    v_add3_u32 v4, v4, v1, s0
-; GFX-950-NEXT:    v_or_b32_e32 v1, 0x400000, v1
-; GFX-950-NEXT:    v_cmp_u_f32_e64 vcc, |v0|, |v0|
-; GFX-950-NEXT:    s_nop 1
-; GFX-950-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX-950-NEXT:    flat_store_short_d16_hi v[2:3], v0
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, |v0|, s0
+; GFX-950-NEXT:    flat_store_short v[2:3], v0
 ; GFX-950-NEXT:    s_endpgm
 entry:
   %a.abs = call float @llvm.fabs.f32(float %a)
@@ -289,15 +310,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_neg(float %a, ptr %out) {
 ; GFX-950:       ; %bb.0: ; %entry
 ; GFX-950-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX-950-NEXT:    v_mov_b32_e32 v2, v1
-; GFX-950-NEXT:    v_xor_b32_e32 v1, 0x80000000, v0
-; GFX-950-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX-950-NEXT:    s_movk_i32 s0, 0x7fff
-; GFX-950-NEXT:    v_add3_u32 v4, v4, v1, s0
-; GFX-950-NEXT:    v_or_b32_e32 v1, 0x400000, v1
-; GFX-950-NEXT:    v_cmp_u_f32_e64 vcc, -v0, -v0
-; GFX-950-NEXT:    s_nop 1
-; GFX-950-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX-950-NEXT:    flat_store_short_d16_hi v[2:3], v0
+; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, -v0, s0
+; GFX-950-NEXT:    flat_store_short v[2:3], v0
 ; GFX-950-NE...
[truncated]

This was referenced Nov 18, 2024

AMDGPU: Add first gfx950 mfma instructions #116312

Merged

AMDGPU: Define v_mfma_f32_32x32x16_bf16 for gfx950 #116679

Open

AMDGPU: Handle gfx950 global_load_lds_* instructions #116680

Open

AMDGPU: Handle gfx950 96/128-bit buffer_load_lds #116681

Open

arsenm added backend:AMDGPU clang Clang issues not falling into any other category clang:frontend Language frontend issues, e.g. anything involving "Sema" llvm:analysis llvm:ir mc Machine (object) code labels Nov 18, 2024 — with Graphite App

arsenm requested review from jayfoad, kosarev, pravinjagtap, rampitec, scchan, shiltian and Sisyph November 18, 2024 19:03

arsenm marked this pull request as ready for review November 18, 2024 19:03

arsenm force-pushed the users/arsenm/gfx950/add-first-mfma-insts branch from 566cdf8 to 6c8fd97 Compare November 18, 2024 20:07

arsenm force-pushed the users/arsenm/gfx950/v_cvt_pk_bf16_f32 branch from f3682aa to 1adfc6b Compare November 18, 2024 20:07

arsenm force-pushed the users/arsenm/gfx950/add-first-mfma-insts branch from 6c8fd97 to d5a560b Compare November 18, 2024 21:35

Base automatically changed from users/arsenm/gfx950/add-first-mfma-insts to main November 18, 2024 21:38

arsenm force-pushed the users/arsenm/gfx950/v_cvt_pk_bf16_f32 branch from 1adfc6b to 2558f35 Compare November 18, 2024 21:39

AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950

deeee03

arsenm force-pushed the users/arsenm/gfx950/v_cvt_pk_bf16_f32 branch from 2558f35 to deeee03 Compare November 19, 2024 01:29

shiltian approved these changes Nov 19, 2024

View reviewed changes

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950 #116678

AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950 #116678

arsenm commented Nov 18, 2024

arsenm commented Nov 18, 2024 •

edited

Loading

llvmbot commented Nov 18, 2024 •

edited

Loading

AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950 #116678

Are you sure you want to change the base?

AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950 #116678

Conversation

arsenm commented Nov 18, 2024

arsenm commented Nov 18, 2024 • edited Loading

llvmbot commented Nov 18, 2024 • edited Loading

arsenm commented Nov 18, 2024 •

edited

Loading

llvmbot commented Nov 18, 2024 •

edited

Loading