Skip to content

Commit 2bb6566

Browse files
fhahnfhahn
committed
[LV] Allow re-processing of operands of instrs feeding interleave group
Follow up to d216615 to update dead interleave group pointer detection to allow re-processing of operands of instructions determined to only feed interleave groups. This is needed because instructions feeding interleave group pointers can become dead in any order, as per the newly added test case.
1 parent 63fae3e commit 2bb6566

File tree

2 files changed

+231
-6
lines changed

2 files changed

+231
-6
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7027,7 +7027,7 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
70277027
// Ignore ephemeral values.
70287028
CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore);
70297029

7030-
SmallVector<Value *> InitialInterleavePointersOps;
7030+
SmallVector<Value *, 4> DeadInterleavePointerOps;
70317031
for (BasicBlock *BB : TheLoop->blocks())
70327032
for (Instruction &I : *BB) {
70337033
// Find all stores to invariant variables. Since they are going to sink
@@ -7045,13 +7045,10 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
70457045
if (Group->getInsertPos() == &I)
70467046
continue;
70477047
Value *PointerOp = getLoadStorePointerOperand(&I);
7048-
InitialInterleavePointersOps.push_back(PointerOp);
7048+
DeadInterleavePointerOps.push_back(PointerOp);
70497049
}
70507050
}
70517051

7052-
SmallSetVector<Value *, 4> DeadInterleavePointerOps(
7053-
InitialInterleavePointersOps.rbegin(),
7054-
InitialInterleavePointersOps.rend());
70557052
// Mark ops feeding interleave group members as free, if they are only used
70567053
// by other dead computations.
70577054
for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
@@ -7064,7 +7061,7 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
70647061
}))
70657062
continue;
70667063
VecValuesToIgnore.insert(Op);
7067-
DeadInterleavePointerOps.insert(Op->op_begin(), Op->op_end());
7064+
DeadInterleavePointerOps.append(Op->op_begin(), Op->op_end());
70687065
}
70697066

70707067
// Ignore type-promoting instructions we identified during reduction

‎llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,230 @@ exit:
373373
ret void
374374
}
375375

376+
define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N) #1 {
377+
; CHECK-LABEL: define void @geps_feeding_interleave_groups_with_reuse2(
378+
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
379+
; CHECK-NEXT: [[ENTRY:.*]]:
380+
; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N]], 3
381+
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
382+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 28
383+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
384+
; CHECK: [[VECTOR_SCEVCHECK]]:
385+
; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[N]], 3
386+
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 24
387+
; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
388+
; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
389+
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
390+
; CHECK-NEXT: [[TMP3:%.*]] = sub i64 0, [[MUL_RESULT]]
391+
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
392+
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP]]
393+
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]]
394+
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 28
395+
; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
396+
; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
397+
; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
398+
; CHECK-NEXT: [[TMP7:%.*]] = sub i64 0, [[MUL_RESULT3]]
399+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT3]]
400+
; CHECK-NEXT: [[TMP9:%.*]] = icmp ult ptr [[TMP8]], [[SCEVGEP1]]
401+
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW4]]
402+
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[A]], i64 20
403+
; CHECK-NEXT: [[MUL6:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
404+
; CHECK-NEXT: [[MUL_RESULT7:%.*]] = extractvalue { i64, i1 } [[MUL6]], 0
405+
; CHECK-NEXT: [[MUL_OVERFLOW8:%.*]] = extractvalue { i64, i1 } [[MUL6]], 1
406+
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 0, [[MUL_RESULT7]]
407+
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP5]], i64 [[MUL_RESULT7]]
408+
; CHECK-NEXT: [[TMP13:%.*]] = icmp ult ptr [[TMP12]], [[SCEVGEP5]]
409+
; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[MUL_OVERFLOW8]]
410+
; CHECK-NEXT: [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[A]], i64 16
411+
; CHECK-NEXT: [[MUL10:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
412+
; CHECK-NEXT: [[MUL_RESULT11:%.*]] = extractvalue { i64, i1 } [[MUL10]], 0
413+
; CHECK-NEXT: [[MUL_OVERFLOW12:%.*]] = extractvalue { i64, i1 } [[MUL10]], 1
414+
; CHECK-NEXT: [[TMP15:%.*]] = sub i64 0, [[MUL_RESULT11]]
415+
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[SCEVGEP9]], i64 [[MUL_RESULT11]]
416+
; CHECK-NEXT: [[TMP17:%.*]] = icmp ult ptr [[TMP16]], [[SCEVGEP9]]
417+
; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP17]], [[MUL_OVERFLOW12]]
418+
; CHECK-NEXT: [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[A]], i64 12
419+
; CHECK-NEXT: [[MUL14:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
420+
; CHECK-NEXT: [[MUL_RESULT15:%.*]] = extractvalue { i64, i1 } [[MUL14]], 0
421+
; CHECK-NEXT: [[MUL_OVERFLOW16:%.*]] = extractvalue { i64, i1 } [[MUL14]], 1
422+
; CHECK-NEXT: [[TMP19:%.*]] = sub i64 0, [[MUL_RESULT15]]
423+
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SCEVGEP13]], i64 [[MUL_RESULT15]]
424+
; CHECK-NEXT: [[TMP21:%.*]] = icmp ult ptr [[TMP20]], [[SCEVGEP13]]
425+
; CHECK-NEXT: [[TMP22:%.*]] = or i1 [[TMP21]], [[MUL_OVERFLOW16]]
426+
; CHECK-NEXT: [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[A]], i64 8
427+
; CHECK-NEXT: [[MUL18:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
428+
; CHECK-NEXT: [[MUL_RESULT19:%.*]] = extractvalue { i64, i1 } [[MUL18]], 0
429+
; CHECK-NEXT: [[MUL_OVERFLOW20:%.*]] = extractvalue { i64, i1 } [[MUL18]], 1
430+
; CHECK-NEXT: [[TMP23:%.*]] = sub i64 0, [[MUL_RESULT19]]
431+
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP17]], i64 [[MUL_RESULT19]]
432+
; CHECK-NEXT: [[TMP25:%.*]] = icmp ult ptr [[TMP24]], [[SCEVGEP17]]
433+
; CHECK-NEXT: [[TMP26:%.*]] = or i1 [[TMP25]], [[MUL_OVERFLOW20]]
434+
; CHECK-NEXT: [[SCEVGEP21:%.*]] = getelementptr i8, ptr [[A]], i64 4
435+
; CHECK-NEXT: [[MUL22:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
436+
; CHECK-NEXT: [[MUL_RESULT23:%.*]] = extractvalue { i64, i1 } [[MUL22]], 0
437+
; CHECK-NEXT: [[MUL_OVERFLOW24:%.*]] = extractvalue { i64, i1 } [[MUL22]], 1
438+
; CHECK-NEXT: [[TMP27:%.*]] = sub i64 0, [[MUL_RESULT23]]
439+
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[SCEVGEP21]], i64 [[MUL_RESULT23]]
440+
; CHECK-NEXT: [[TMP29:%.*]] = icmp ult ptr [[TMP28]], [[SCEVGEP21]]
441+
; CHECK-NEXT: [[TMP30:%.*]] = or i1 [[TMP29]], [[MUL_OVERFLOW24]]
442+
; CHECK-NEXT: [[MUL25:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
443+
; CHECK-NEXT: [[MUL_RESULT26:%.*]] = extractvalue { i64, i1 } [[MUL25]], 0
444+
; CHECK-NEXT: [[MUL_OVERFLOW27:%.*]] = extractvalue { i64, i1 } [[MUL25]], 1
445+
; CHECK-NEXT: [[TMP31:%.*]] = sub i64 0, [[MUL_RESULT26]]
446+
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[A]], i64 [[MUL_RESULT26]]
447+
; CHECK-NEXT: [[TMP33:%.*]] = icmp ult ptr [[TMP32]], [[A]]
448+
; CHECK-NEXT: [[TMP34:%.*]] = or i1 [[TMP33]], [[MUL_OVERFLOW27]]
449+
; CHECK-NEXT: [[TMP35:%.*]] = or i1 [[TMP6]], [[TMP10]]
450+
; CHECK-NEXT: [[TMP36:%.*]] = or i1 [[TMP35]], [[TMP14]]
451+
; CHECK-NEXT: [[TMP37:%.*]] = or i1 [[TMP36]], [[TMP18]]
452+
; CHECK-NEXT: [[TMP38:%.*]] = or i1 [[TMP37]], [[TMP22]]
453+
; CHECK-NEXT: [[TMP39:%.*]] = or i1 [[TMP38]], [[TMP26]]
454+
; CHECK-NEXT: [[TMP40:%.*]] = or i1 [[TMP39]], [[TMP30]]
455+
; CHECK-NEXT: [[TMP41:%.*]] = or i1 [[TMP40]], [[TMP34]]
456+
; CHECK-NEXT: br i1 [[TMP41]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
457+
; CHECK: [[VECTOR_MEMCHECK]]:
458+
; CHECK-NEXT: [[TMP42:%.*]] = lshr i64 [[N]], 3
459+
; CHECK-NEXT: [[TMP43:%.*]] = shl i64 [[TMP42]], 5
460+
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP43]], 32
461+
; CHECK-NEXT: [[SCEVGEP28:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP44]]
462+
; CHECK-NEXT: [[TMP45:%.*]] = add nuw nsw i64 [[TMP43]], 4
463+
; CHECK-NEXT: [[SCEVGEP29:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP45]]
464+
; CHECK-NEXT: [[TMP46:%.*]] = shl i64 [[TMP42]], 4
465+
; CHECK-NEXT: [[TMP47:%.*]] = add nuw nsw i64 [[TMP46]], 8
466+
; CHECK-NEXT: [[SCEVGEP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP47]]
467+
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP29]]
468+
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP28]]
469+
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
470+
; CHECK-NEXT: [[BOUND031:%.*]] = icmp ult ptr [[A]], [[SCEVGEP30]]
471+
; CHECK-NEXT: [[BOUND132:%.*]] = icmp ult ptr [[B]], [[SCEVGEP28]]
472+
; CHECK-NEXT: [[FOUND_CONFLICT33:%.*]] = and i1 [[BOUND031]], [[BOUND132]]
473+
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT33]]
474+
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
475+
; CHECK: [[VECTOR_PH]]:
476+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
477+
; CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
478+
; CHECK-NEXT: [[TMP49:%.*]] = select i1 [[TMP48]], i64 4, i64 [[N_MOD_VF]]
479+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP49]]
480+
; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 8
481+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
482+
; CHECK: [[VECTOR_BODY]]:
483+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
484+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 8, i64 16, i64 24>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
485+
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
486+
; CHECK-NEXT: [[TMP50:%.*]] = add i64 [[OFFSET_IDX]], 0
487+
; CHECK-NEXT: [[TMP51:%.*]] = lshr exact i64 [[TMP50]], 1
488+
; CHECK-NEXT: [[TMP52:%.*]] = getelementptr i32, ptr [[B]], i64 [[TMP51]]
489+
; CHECK-NEXT: [[TMP53:%.*]] = getelementptr i32, ptr [[TMP52]], i32 0
490+
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP53]], align 4
491+
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
492+
; CHECK-NEXT: [[STRIDED_VEC34:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
493+
; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[B]], <4 x i64> [[VEC_IND]]
494+
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP54]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> poison), !alias.scope [[META6:![0-9]+]]
495+
; CHECK-NEXT: [[TMP55:%.*]] = or disjoint i64 [[TMP50]], 7
496+
; CHECK-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP55]]
497+
; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP56]], i32 -7
498+
; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
499+
; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC34]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
500+
; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_GATHER]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
501+
; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <8 x i32> [[TMP58]], <8 x i32> [[TMP59]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
502+
; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <8 x i32> [[TMP60]], <8 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
503+
; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP61]], <16 x i32> [[TMP62]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
504+
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[TMP63]], <32 x i32> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
505+
; CHECK-NEXT: store <32 x i32> [[INTERLEAVED_VEC]], ptr [[TMP57]], align 4
506+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
507+
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 32, i64 32, i64 32, i64 32>
508+
; CHECK-NEXT: [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
509+
; CHECK-NEXT: br i1 [[TMP64]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
510+
; CHECK: [[MIDDLE_BLOCK]]:
511+
; CHECK-NEXT: br label %[[SCALAR_PH]]
512+
; CHECK: [[SCALAR_PH]]:
513+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
514+
; CHECK-NEXT: br label %[[LOOP:.*]]
515+
; CHECK: [[LOOP]]:
516+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_7:%.*]], %[[LOOP]] ]
517+
; CHECK-NEXT: [[SHR_1:%.*]] = lshr exact i64 [[IV]], 1
518+
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr nusw i32, ptr [[B]], i64 [[SHR_1]]
519+
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_B]], align 4
520+
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
521+
; CHECK-NEXT: store i32 [[L]], ptr [[GEP_A]], align 4
522+
; CHECK-NEXT: [[IV_NEXT:%.*]] = or disjoint i64 [[IV]], 1
523+
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT]]
524+
; CHECK-NEXT: store i32 0, ptr [[GEP_A_1]], align 4
525+
; CHECK-NEXT: [[IV_NEXT_1:%.*]] = or disjoint i64 [[IV]], 2
526+
; CHECK-NEXT: [[SHR_2:%.*]] = lshr exact i64 [[IV_NEXT_1]], 1
527+
; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr i32, ptr [[B]], i64 [[SHR_2]]
528+
; CHECK-NEXT: [[TMP65:%.*]] = load i32, ptr [[GEP_B_2]], align 4
529+
; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_1]]
530+
; CHECK-NEXT: store i32 [[TMP65]], ptr [[GEP_A_2]], align 4
531+
; CHECK-NEXT: [[IV_NEXT_2:%.*]] = or disjoint i64 [[IV]], 3
532+
; CHECK-NEXT: [[GEP_A_3:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_2]]
533+
; CHECK-NEXT: store i32 0, ptr [[GEP_A_3]], align 4
534+
; CHECK-NEXT: [[IV_NEXT_3:%.*]] = or disjoint i64 [[IV]], 4
535+
; CHECK-NEXT: [[GEP_B_4:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]
536+
; CHECK-NEXT: [[TMP66:%.*]] = load i32, ptr [[GEP_B_4]], align 4
537+
; CHECK-NEXT: [[GEP_A_4:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_3]]
538+
; CHECK-NEXT: store i32 [[TMP66]], ptr [[GEP_A_4]], align 4
539+
; CHECK-NEXT: [[IV_NEXT_4:%.*]] = or disjoint i64 [[IV]], 5
540+
; CHECK-NEXT: [[GEP_A_5:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_4]]
541+
; CHECK-NEXT: store i32 0, ptr [[GEP_A_5]], align 4
542+
; CHECK-NEXT: [[IV_NEXT_5:%.*]] = or disjoint i64 [[IV]], 6
543+
; CHECK-NEXT: [[GEP_A_6:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_5]]
544+
; CHECK-NEXT: store i32 0, ptr [[GEP_A_6]], align 4
545+
; CHECK-NEXT: [[IV_NEXT_6:%.*]] = or disjoint i64 [[IV]], 7
546+
; CHECK-NEXT: [[GEP_A_7:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_6]]
547+
; CHECK-NEXT: store i32 0, ptr [[GEP_A_7]], align 4
548+
; CHECK-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8
549+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
550+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
551+
; CHECK: [[EXIT]]:
552+
; CHECK-NEXT: ret void
553+
;
554+
entry:
555+
br label %loop
556+
557+
loop:
558+
%iv = phi i64 [ 0, %entry ], [ %iv.next.7, %loop ]
559+
%shr.1 = lshr exact i64 %iv, 1
560+
%gep.B = getelementptr nusw i32, ptr %B, i64 %shr.1
561+
%l = load i32, ptr %gep.B, align 4
562+
%gep.A = getelementptr i32, ptr %A, i64 %iv
563+
store i32 %l, ptr %gep.A, align 4
564+
%iv.next = or disjoint i64 %iv, 1
565+
%gep.A.1 = getelementptr i32, ptr %A, i64 %iv.next
566+
store i32 0, ptr %gep.A.1, align 4
567+
%iv.next.1 = or disjoint i64 %iv, 2
568+
%shr.2 = lshr exact i64 %iv.next.1, 1
569+
%gep.B.2 = getelementptr i32, ptr %B, i64 %shr.2
570+
%1 = load i32, ptr %gep.B.2, align 4
571+
%gep.A.2 = getelementptr i32, ptr %A, i64 %iv.next.1
572+
store i32 %1, ptr %gep.A.2, align 4
573+
%iv.next.2 = or disjoint i64 %iv, 3
574+
%gep.A.3 = getelementptr i32, ptr %A, i64 %iv.next.2
575+
store i32 0, ptr %gep.A.3, align 4
576+
%iv.next.3 = or disjoint i64 %iv, 4
577+
%gep.B.4 = getelementptr i32, ptr %B, i64 %iv
578+
%2 = load i32, ptr %gep.B.4, align 4
579+
%gep.A.4 = getelementptr i32, ptr %A, i64 %iv.next.3
580+
store i32 %2, ptr %gep.A.4, align 4
581+
%iv.next.4 = or disjoint i64 %iv, 5
582+
%gep.A.5 = getelementptr i32, ptr %A, i64 %iv.next.4
583+
store i32 0, ptr %gep.A.5, align 4
584+
%iv.next.5 = or disjoint i64 %iv, 6
585+
%gep.A.6 = getelementptr i32, ptr %A, i64 %iv.next.5
586+
store i32 0, ptr %gep.A.6, align 4
587+
%iv.next.6 = or disjoint i64 %iv, 7
588+
%gep.A.7 = getelementptr i32, ptr %A, i64 %iv.next.6
589+
store i32 0, ptr %gep.A.7, align 4
590+
%iv.next.7 = add nuw nsw i64 %iv, 8
591+
%ec = icmp eq i64 %iv, %N
592+
br i1 %ec, label %exit, label %loop
593+
594+
exit:
595+
ret void
596+
}
597+
376598
attributes #0 = { "target-features"="+sse4.2" }
599+
attributes #1 = { "min-legal-vector-width"="0" "target-cpu"="cascadelake" }
377600

378601
;.
379602
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
@@ -382,4 +605,9 @@ attributes #0 = { "target-features"="+sse4.2" }
382605
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
383606
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
384607
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
608+
; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
609+
; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
610+
; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
611+
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
612+
; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]]}
385613
;.

0 commit comments

Comments
 (0)