Skip to content

[LLVM][SVE] Add isel for bfloat based (de)interleave operations. #128875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 27, 2025
Merged

[LLVM][SVE] Add isel for bfloat based (de)interleave operations. #128875

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4fba5d3
[LLVM][SVE] Add isel for bfloat based (de)interleave operations.
paulwalker-arm Feb 26, 2025
b83d3f3
Fix whitespace formating within unit tests.
paulwalker-arm Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1702,6 +1702,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::MLOAD, VT, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);

if (Subtarget->hasSVEB16B16()) {
Expand Down
2 changes: 2 additions & 0 deletions llvm/lib/Target/AArch64/SVEInstrFormats.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3079,6 +3079,8 @@ multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm,
def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;

def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4bf16, op, nxv4bf16, nxv4bf16, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2bf16, op, nxv2bf16, nxv2bf16, !cast<Instruction>(NAME # _D)>;
}

//===----------------------------------------------------------------------===//
Expand Down
85 changes: 60 additions & 25 deletions llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ define {<vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_n
; CHECK-NEXT: uunpklo z1.d, z2.s
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x half>, <vscale x 2 x half>} @llvm.vector.deinterleave2.nxv4f16(<vscale x 4 x half> %vec)
ret {<vscale x 2 x half>, <vscale x 2 x half>} %retval
ret {<vscale x 2 x half>, <vscale x 2 x half>} %retval
}

define {<vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_nxv8f16(<vscale x 8 x half> %vec) {
Expand All @@ -22,7 +22,7 @@ define {<vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_n
; CHECK-NEXT: uunpklo z1.s, z2.h
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave2.nxv8f16(<vscale x 8 x half> %vec)
ret {<vscale x 4 x half>, <vscale x 4 x half>} %retval
ret {<vscale x 4 x half>, <vscale x 4 x half>} %retval
}

define {<vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_nxv16f16(<vscale x 16 x half> %vec) {
Expand All @@ -33,7 +33,7 @@ define {<vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_n
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.vector.deinterleave2.nxv16f16(<vscale x 16 x half> %vec)
ret {<vscale x 8 x half>, <vscale x 8 x half>} %retval
ret {<vscale x 8 x half>, <vscale x 8 x half>} %retval
}

define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32_nxv4f32(<vscale x 4 x float> %vec) {
Expand All @@ -45,7 +45,7 @@ define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32
; CHECK-NEXT: uunpklo z1.d, z2.s
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float> %vec)
ret {<vscale x 2 x float>, <vscale x 2 x float>} %retval
ret {<vscale x 2 x float>, <vscale x 2 x float>} %retval
}

define {<vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32_nxv8f32(<vscale x 8 x float> %vec) {
Expand All @@ -56,7 +56,7 @@ define {<vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %vec)
ret {<vscale x 4 x float>, <vscale x 4 x float>} %retval
ret {<vscale x 4 x float>, <vscale x 4 x float>} %retval
}

define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f64_nxv4f64(<vscale x 4 x double> %vec) {
Expand All @@ -67,7 +67,42 @@ define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %vec)
ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
}

define {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @vector_deinterleave_nxv2bf16_nxv4bf16(<vscale x 4 x bfloat> %vec) {
; CHECK-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
; CHECK-NEXT: uzp2 z2.s, z0.s, z0.s
; CHECK-NEXT: uunpklo z0.d, z1.s
; CHECK-NEXT: uunpklo z1.d, z2.s
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @llvm.vector.deinterleave2.nxv4bf16(<vscale x 4 x bfloat> %vec)
ret {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} %retval
}

define {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @vector_deinterleave_nxv4bf16_nxv8bf16(<vscale x 8 x bfloat> %vec) {
; CHECK-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: uzp1 z1.h, z0.h, z0.h
; CHECK-NEXT: uzp2 z2.h, z0.h, z0.h
; CHECK-NEXT: uunpklo z0.s, z1.h
; CHECK-NEXT: uunpklo z1.s, z2.h
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @llvm.vector.deinterleave2.nxv8bf16(<vscale x 8 x bfloat> %vec)
ret {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} %retval
}

define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @vector_deinterleave_nxv8bf16_nxv16bf16(<vscale x 16 x bfloat> %vec) {
; CHECK-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: uzp1 z2.h, z0.h, z1.h
; CHECK-NEXT: uzp2 z1.h, z0.h, z1.h
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.vector.deinterleave2.nxv16bf16(<vscale x 16 x bfloat> %vec)
ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %retval
}

; Integers
Expand All @@ -80,7 +115,7 @@ define {<vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)
ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval
ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval
}

define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv16i16(<vscale x 16 x i16> %vec) {
Expand All @@ -91,7 +126,7 @@ define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> %vec)
ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %retval
ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %retval
}

define {<vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxvv8i32(<vscale x 8 x i32> %vec) {
Expand All @@ -102,7 +137,7 @@ define {<vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %vec)
ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval
ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval
}

define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv4i64(<vscale x 4 x i64> %vec) {
Expand All @@ -113,7 +148,7 @@ define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv
; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %vec)
ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval
ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval
}

; Predicated
Expand All @@ -125,7 +160,7 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv
; CHECK-NEXT: mov p0.b, p2.b
; CHECK-NEXT: ret
%retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)
ret {<vscale x 16 x i1>, <vscale x 16 x i1>} %retval
ret {<vscale x 16 x i1>, <vscale x 16 x i1>} %retval
}

define {<vscale x 8 x i1>, <vscale x 8 x i1>} @vector_deinterleave_nxv8i1_nxv16i1(<vscale x 16 x i1> %vec) {
Expand All @@ -137,7 +172,7 @@ define {<vscale x 8 x i1>, <vscale x 8 x i1>} @vector_deinterleave_nxv8i1_nxv16i
; CHECK-NEXT: punpklo p1.h, p2.b
; CHECK-NEXT: ret
%retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave2.nxv16i1(<vscale x 16 x i1> %vec)
ret {<vscale x 8 x i1>, <vscale x 8 x i1>} %retval
ret {<vscale x 8 x i1>, <vscale x 8 x i1>} %retval
}

define {<vscale x 4 x i1>, <vscale x 4 x i1>} @vector_deinterleave_nxv4i1_nxv8i1(<vscale x 8 x i1> %vec) {
Expand All @@ -149,7 +184,7 @@ define {<vscale x 4 x i1>, <vscale x 4 x i1>} @vector_deinterleave_nxv4i1_nxv8i1
; CHECK-NEXT: punpklo p1.h, p2.b
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x i1>, <vscale x 4 x i1>} @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %vec)
ret {<vscale x 4 x i1>, <vscale x 4 x i1>} %retval
ret {<vscale x 4 x i1>, <vscale x 4 x i1>} %retval
}

define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1(<vscale x 4 x i1> %vec) {
Expand All @@ -161,7 +196,7 @@ define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1
; CHECK-NEXT: punpklo p1.h, p2.b
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x i1>, <vscale x 2 x i1>} @llvm.vector.deinterleave2.nxv4i1(<vscale x 4 x i1> %vec)
ret {<vscale x 2 x i1>, <vscale x 2 x i1>} %retval
ret {<vscale x 2 x i1>, <vscale x 2 x i1>} %retval
}


Expand All @@ -178,11 +213,11 @@ define {<vscale x 4 x i64>, <vscale x 4 x i64>} @vector_deinterleave_nxv4i64_nxv
; CHECK-NEXT: mov z1.d, z4.d
; CHECK-NEXT: mov z2.d, z6.d
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x i64>, <vscale x 4 x i64>} @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> %vec)
ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval
%retval = call {<vscale x 4 x i64>, <vscale x 4 x i64>} @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> %vec)
ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval
}

define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv16i64(<vscale x 16 x i64> %vec) {
define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv16i64(<vscale x 16 x i64> %vec) {
; CHECK-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
; CHECK: // %bb.0:
; CHECK-NEXT: uzp1 z24.d, z2.d, z3.d
Expand All @@ -201,8 +236,8 @@ define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nx
; CHECK-NEXT: mov z5.d, z29.d
; CHECK-NEXT: mov z6.d, z30.d
; CHECK-NEXT: ret
%retval = call {<vscale x 8 x i64>, <vscale x 8 x i64>} @llvm.vector.deinterleave2.nxv16i64(<vscale x 16 x i64> %vec)
ret {<vscale x 8 x i64>, <vscale x 8 x i64>} %retval
%retval = call {<vscale x 8 x i64>, <vscale x 8 x i64>} @llvm.vector.deinterleave2.nxv16i64(<vscale x 16 x i64> %vec)
ret {<vscale x 8 x i64>, <vscale x 8 x i64>} %retval
}


Expand All @@ -216,8 +251,8 @@ define {<vscale x 8 x i8>, <vscale x 8 x i8>} @vector_deinterleave_nxv8i8_nxv16i
; CHECK-NEXT: uzp1 z0.h, z2.h, z1.h
; CHECK-NEXT: uzp2 z1.h, z2.h, z1.h
; CHECK-NEXT: ret
%retval = call {<vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> %vec)
ret {<vscale x 8 x i8>, <vscale x 8 x i8>} %retval
%retval = call {<vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> %vec)
ret {<vscale x 8 x i8>, <vscale x 8 x i8>} %retval
}

define {<vscale x 4 x i16>, <vscale x 4 x i16>} @vector_deinterleave_nxv4i16_nxv8i16(<vscale x 8 x i16> %vec) {
Expand All @@ -228,8 +263,8 @@ define {<vscale x 4 x i16>, <vscale x 4 x i16>} @vector_deinterleave_nxv4i16_nxv
; CHECK-NEXT: uzp1 z0.s, z2.s, z1.s
; CHECK-NEXT: uzp2 z1.s, z2.s, z1.s
; CHECK-NEXT: ret
%retval = call {<vscale x 4 x i16>, <vscale x 4 x i16>} @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %vec)
ret {<vscale x 4 x i16>, <vscale x 4 x i16>} %retval
%retval = call {<vscale x 4 x i16>, <vscale x 4 x i16>} @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %vec)
ret {<vscale x 4 x i16>, <vscale x 4 x i16>} %retval
}

define {<vscale x 2 x i32>, <vscale x 2 x i32>} @vector_deinterleave_nxv2i32_nxv4i32(<vscale x 4 x i32> %vec) {
Expand All @@ -240,8 +275,8 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>} @vector_deinterleave_nxv2i32_nxv
; CHECK-NEXT: uzp1 z0.d, z2.d, z1.d
; CHECK-NEXT: uzp2 z1.d, z2.d, z1.d
; CHECK-NEXT: ret
%retval = call {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %vec)
ret {<vscale x 2 x i32>, <vscale x 2 x i32>} %retval
%retval = call {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %vec)
ret {<vscale x 2 x i32>, <vscale x 2 x i32>} %retval
}


Expand Down
Loading