Skip to content

Commit

Permalink
Merge pull request #5061 from XiWeiGu/la64_update_symv
Browse files Browse the repository at this point in the history 
LoongArch64: Update symv
  • Loading branch information
@martin-frbg
martin-frbg authored Jan 10, 2025
2 parents a588ea9 + 20a8e48 commit c31f148
Show file tree
Hide file tree
Showing 4 changed files with 444 additions and 383 deletions.
232 changes: 127 additions & 105 deletions kernel/loongarch64/dsymv_L_lasx.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define ASSEMBLER

#include "common.h"
#include "loongarch64_asm.S"

/* Param */
#define M $r4
Expand Down Expand Up @@ -57,6 +58,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define T2 $r28
#define T3 $r29
#define T4 $r30
#define T5 $r17
#define T6 $r16

/* LSX vectors */
#define U0 $xr31
Expand Down Expand Up @@ -87,10 +90,113 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define a8 $f8
#define a9 $f9

.macro LOAD_Y_8
beqz T5, .L01_Y_0
add.d T2, IY, INCY
fldx.d $f4, Y, T2
add.d T2, T2, INCY
fldx.d $f5, Y, T2
add.d T2, T2, INCY
fldx.d $f6, Y, T2
add.d T2, T2, INCY
fldx.d $f7, Y, T2

PROLOGUE
add.d T2, T2, INCY
fldx.d $f8, Y, T2
add.d T2, T2, INCY
fldx.d $f9, Y, T2
add.d T2, T2, INCY
fldx.d $f10, Y, T2
add.d T2, T2, INCY
fldx.d $f11, Y, T2

vextrins.d $vr4, $vr5, 0x10
vextrins.d $vr6, $vr7, 0x10
xvpermi.q U4, U6, 0x02

LDARG BUFFER, $sp, 0
vextrins.d $vr8, $vr9, 0x10
vextrins.d $vr10, $vr11, 0x10
xvpermi.q U8, U10, 0x02
b .L01_Y_1
.L01_Y_0:
add.d T3, IY, INCY
xvldx U4, Y, T3
alsl.d T4, INCY, T3, 2
xvldx U8, Y, T4
.L01_Y_1:
.endm

.macro LOAD_X_8
beqz T6, .L01_X_0
add.d T2, IX, INCX
fldx.d $f4, X, T2
add.d T2, T2, INCX
fldx.d $f5, X, T2
add.d T2, T2, INCX
fldx.d $f6, X, T2
add.d T2, T2, INCX
fldx.d $f7, X, T2

add.d T2, T2, INCX
fldx.d $f8, X, T2
add.d T2, T2, INCX
fldx.d $f9, X, T2
add.d T2, T2, INCX
fldx.d $f10, X, T2
add.d T2, T2, INCX
fldx.d $f11, X, T2

vextrins.d $vr4, $vr5, 0x10
vextrins.d $vr6, $vr7, 0x10
xvpermi.q U4, U6, 0x02

vextrins.d $vr8, $vr9, 0x10
vextrins.d $vr10, $vr11, 0x10
xvpermi.q U8, U10, 0x02
b .L01_X_1
.L01_X_0:
add.d T3, IX, INCX
xvldx U4, X, T3
alsl.d T2, INCX, T3, 2
xvldx U8, X, T2
.L01_X_1:
.endm

.macro STORE_Y_8
beqz T5, .L01_Y_2
xvpermi.d U6, U4, 0xee
vextrins.d $vr5, $vr4, 0x01
vextrins.d $vr7, $vr6, 0x01

xvpermi.d U10, U8, 0xee
vextrins.d $vr9, $vr8, 0x01
vextrins.d $vr11, $vr10, 0x01

add.d T2, IY, INCY
fstx.d $f4, Y, T2
add.d T2, T2, INCY
fstx.d $f5, Y, T2
add.d T2, T2, INCY
fstx.d $f6, Y, T2
add.d T2, T2, INCY
fstx.d $f7, Y, T2

add.d T2, T2, INCY
fstx.d $f8, Y, T2
add.d T2, T2, INCY
fstx.d $f9, Y, T2
add.d T2, T2, INCY
fstx.d $f10, Y, T2
add.d T2, T2, INCY
fstx.d $f11, Y, T2
b .L01_Y_3
.L01_Y_2:
xvstx U4, Y, T3
xvstx U8, Y, T4
.L01_Y_3:
.endm

PROLOGUE

addi.d $sp, $sp, -88

Expand All @@ -107,6 +213,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

xvldrepl.d VALPHA, $sp, 80

addi.d T5, INCY, -1
addi.d T6, INCX, -1
slli.d LDA, LDA, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
slli.d INCY, INCY, BASE_SHIFT
Expand All @@ -122,11 +230,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
beq J, N, .L999

.L01:
MTC a2, $r0 //temp2
xvxor.v U2, U2, U2
fldx.d a6, X, JX
fmul.d a3, ALPHA, a6 //temp1
xvreplve0.d U3, U3
xvreplve0.d U2, U2

mul.d T0, J, LDA
slli.d T1, J, BASE_SHIFT
Expand All @@ -147,126 +254,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
srai.d T0, T0, 3
add.d T0, T0, J
addi.d T0, T0, 1
beq I, T0, .L03
bge I, T0, .L03
beq I, T0, .L03
bge I, T0, .L03

mul.d T1, J, LDA
add.d T1, T1, II

.L02: /* /8 */
xvldx U1, AO1, T1
addi.d T1, T1, 32
xvldx U14, AO1, T1
addi.d T1, T1, 32
addi.d T2, T1, 32
xvldx U14, AO1, T2

add.d T2, IY, INCY
fldx.d $f4, Y, T2
add.d T2, T2, INCY
fldx.d $f5, Y, T2
add.d T2, T2, INCY
fldx.d $f6, Y, T2
add.d T2, T2, INCY
fldx.d $f7, Y, T2

add.d T2, T2, INCY
fldx.d $f8, Y, T2
add.d T2, T2, INCY
fldx.d $f9, Y, T2
add.d T2, T2, INCY
fldx.d $f10, Y, T2
add.d T2, T2, INCY
fldx.d $f11, Y, T2

vextrins.d $vr4, $vr5, 0x10
vextrins.d $vr6, $vr7, 0x10
xvpermi.q U4, U6, 0x02

vextrins.d $vr8, $vr9, 0x10
vextrins.d $vr10, $vr11, 0x10
xvpermi.q U8, U10, 0x02
LOAD_Y_8

xvfmadd.d U4, U3, U1, U4
xvfmadd.d U8, U3, U14, U8

xvpermi.d U6, U4, 0xee
vextrins.d $vr5, $vr4, 0x01
vextrins.d $vr7, $vr6, 0x01

xvpermi.d U10, U8, 0xee
vextrins.d $vr9, $vr8, 0x01
vextrins.d $vr11, $vr10, 0x01

add.d T2, IY, INCY
fstx.d $f4, Y, T2
add.d T2, T2, INCY
fstx.d $f5, Y, T2
add.d T2, T2, INCY
fstx.d $f6, Y, T2
add.d T2, T2, INCY
fstx.d $f7, Y, T2

add.d T2, T2, INCY
fstx.d $f8, Y, T2
add.d T2, T2, INCY
fstx.d $f9, Y, T2
add.d T2, T2, INCY
fstx.d $f10, Y, T2
add.d T2, T2, INCY
fstx.d $f11, Y, T2

slli.d T2, INCY, 3
add.d IY, IY, T2

add.d T2, IX, INCX
fldx.d $f4, X, T2
add.d T2, T2, INCX
fldx.d $f5, X, T2
add.d T2, T2, INCX
fldx.d $f6, X, T2
add.d T2, T2, INCX
fldx.d $f7, X, T2

add.d T2, T2, INCX
fldx.d $f8, X, T2
add.d T2, T2, INCX
fldx.d $f9, X, T2
add.d T2, T2, INCX
fldx.d $f10, X, T2
add.d T2, T2, INCX
fldx.d $f11, X, T2

vextrins.d $vr4, $vr5, 0x10
vextrins.d $vr6, $vr7, 0x10
xvpermi.q U4, U6, 0x02

vextrins.d $vr8, $vr9, 0x10
vextrins.d $vr10, $vr11, 0x10
xvpermi.q U8, U10, 0x02

xvand.v $xr12, $xr2, $xr2

xvfmadd.d U2, U1, U4, U2
xvfsub.d U2, U2, $xr12
xvfmadd.d U2, U14, U8, U2
STORE_Y_8

xvpermi.d U4, U2, 0x01
xvpermi.d U5, U2, 0x02
xvpermi.d U6, U2, 0x03
alsl.d IY, INCY, IY, 3

fadd.d $f2, $f2, $f4
fadd.d $f2, $f2, $f5
fadd.d $f2, $f2, $f6
fadd.d $f2, $f2, $f12
LOAD_X_8

xvreplve0.d U2, U2
xvfmadd.d U2, U1, U4, U2
xvfmadd.d U2, U14, U8, U2

slli.d T2, INCX, 3
add.d IX, IX, T2
alsl.d IX, INCX, IX, 3

addi.d T1, T1, 64
addi.d II, II, 64
addi.d I, I, 1
blt I, T0, .L02

//Acc U2
GACC xvf, d, U4, U2
fmov.d $f2, $f4
.L03: /* &4 */
sub.d T0, M, J
addi.d T0, T0, -1
Expand Down Expand Up @@ -437,4 +459,4 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
addi.d $sp, $sp, 88
jirl $r0, $r1, 0x0

EPILOGUE
EPILOGUE
Loading

0 comments on commit c31f148

Please sign in to comment.