commit adb0126ef11b0083d5a78be1534ccefa53def1cf
Author: Patrick O'Neill <patrick@rivosinc.com>
Date:   Tue Apr 23 07:48:43 2024 -0700

    [VPlan] Add scalar inferencing support for Not and Or insns (#89160)

    Fixes #87394.

    PR: https://github.com/llvm/llvm-project/pull/89160

diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index ad4ea648cd61..5f93339083f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -35,6 +35,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     CachedTypes[OtherV] = ResTy;
     return ResTy;
   }
+  case Instruction::Or:
   case Instruction::ICmp:
   case VPInstruction::FirstOrderRecurrenceSplice: {
     Type *ResTy = inferScalarType(R->getOperand(0));
@@ -44,6 +45,12 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     CachedTypes[OtherV] = ResTy;
     return ResTy;
   }
+  case VPInstruction::Not: {
+    Type *ResTy = inferScalarType(R->getOperand(0));
+    assert(IntegerType::get(Ctx, 1) == ResTy &&
+           "unexpected scalar type inferred for operand");
+    return ResTy;
+  }
   case VPInstruction::PtrAdd:
     // Return the type based on the pointer argument (i.e. first operand).
     return inferScalarType(R->getOperand(0));

define void @foo(i8 %arg.0, i8 %arg.1) {
entry:
  br label %loop.header

loop.header:                                      ; preds = %latch, %entry
  %incrementor = phi i8 [ %add, %latch ], [ 0, %entry ]
  %and = and i8 %arg.0, %arg.1
  %extract.t = trunc i8 %and to i1
  br i1 %extract.t, label %latch, label %indirect.latch

indirect.latch:                                   ; preds = %loop.header
  br label %latch

latch:                                            ; preds = %indirect.latch, %loop.header
  %add = add i8 %incrementor, 1
  %conv = zext i8 %incrementor to i32
  %cmp = icmp ult i32 %conv, 1
  br i1 %cmp, label %loop.header, label %loop.exit

loop.exit:                                        ; preds = %latch
  ret void
}

[LoopVectorize][VPlan] Unreachable executed "Unhandled opcode!"

Reduced LLVM IR:
```llvm ir
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
target triple = "riscv64-unknown-linux-gnu"

define i32 @main() #0 {
entry:
  br label %for.body

for.cond.cleanup.loopexit:                        ; preds = %cond.end
  ret i32 0

for.body:                                         ; preds = %cond.end, %entry
  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %cond.end ]
  %conv3 = zext i1 false to i64
  %conv4 = trunc i64 %conv3 to i16
  br i1 false, label %cond.false, label %cond.end

cond.false:                                       ; preds = %for.body
  br label %cond.end

cond.end:                                         ; preds = %cond.false, %for.body
  %indvars.iv.next = add i64 %indvars.iv, 1
  %cmp = icmp ult i64 %indvars.iv, 1
  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
}

attributes #0 = { "target-features"="+64bit,+v" }
```

Backtrace:
```
Unhandled opcode!
UNREACHABLE executed at /scratch/tc-testing/tc-apr-2/llvm/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp:58!
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.      Program arguments: /scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt --passes=loop-vectorize reduced.ll
 #0 0x00005f05c836eb60 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x2d84b60)
 #1 0x00005f05c836bf6f llvm::sys::RunSignalHandlers() (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x2d81f6f)
 #2 0x00005f05c836c0c5 SignalHandler(int) Signals.cpp:0:0
 #3 0x000071a702442520 (/lib/x86_64-linux-gnu/libc.so.6+0x42520)
 #4 0x000071a7024969fc __pthread_kill_implementation ./nptl/pthread_kill.c:44:76
 #5 0x000071a7024969fc __pthread_kill_internal ./nptl/pthread_kill.c:78:10
 #6 0x000071a7024969fc pthread_kill ./nptl/pthread_kill.c:89:10
 #7 0x000071a702442476 gsignal ./signal/../sysdeps/posix/raise.c:27:6
 #8 0x000071a7024287f3 abort ./stdlib/abort.c:81:7
 #9 0x00005f05c82a2afe (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x2cb8afe)
#10 0x00005f05c74dfe67 llvm::VPTypeAnalysis::inferScalarTypeForRecipe(llvm::VPInstruction const*) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1ef5e67)
#11 0x00005f05c74dc88c llvm::VPTypeAnalysis::inferScalarType(llvm::VPValue const*) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1ef288c)
#12 0x00005f05c750790e simplifyRecipes(llvm::VPlan&, llvm::LLVMContext&) VPlanTransforms.cpp:0:0
#13 0x00005f05c750e606 llvm::VPlanTransforms::optimize(llvm::VPlan&, llvm::ScalarEvolution&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1f24606)
#14 0x00005f05c73ded66 llvm::LoopVectorizationPlanner::buildVPlansWithVPRecipes(llvm::ElementCount, llvm::ElementCount) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1df4d66)
#15 0x00005f05c73e5319 llvm::LoopVectorizationPlanner::plan(llvm::ElementCount, unsigned int) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1dfb319)
#16 0x00005f05c73e88a9 llvm::LoopVectorizePass::processLoop(llvm::Loop*) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1dfe8a9)
#17 0x00005f05c73eb73e llvm::LoopVectorizePass::runImpl(llvm::Function&, llvm::ScalarEvolution&, llvm::LoopInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::BlockFrequencyInfo*, llvm::TargetLibraryInfo*, llvm::DemandedBits&, llvm::AssumptionCache&, llvm::LoopAccessInfoManager&, llvm::OptimizationRemarkEmitter&, llvm::ProfileSummaryInfo*) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1e0173e)
#18 0x00005f05c73ec88d llvm::LoopVectorizePass::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x1e0288d)
#19 0x00005f05c62f3276 llvm::detail::PassModel<llvm::Function, llvm::LoopVectorizePass, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0xd09276)
#20 0x00005f05c8195141 llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x2bab141)
#21 0x00005f05c62eabd6 llvm::detail::PassModel<llvm::Function, llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0xd00bd6)
#22 0x00005f05c8193e3b llvm::ModuleToFunctionPassAdaptor::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x2ba9e3b)
#23 0x00005f05c62f2dc6 llvm::detail::PassModel<llvm::Module, llvm::ModuleToFunctionPassAdaptor, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0xd08dc6)
#24 0x00005f05c8191cb1 llvm::PassManager<llvm::Module, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x2ba7cb1)
#25 0x00005f05c5b6cce5 llvm::runPassPipeline(llvm::StringRef, llvm::Module&, llvm::TargetMachine*, llvm::TargetLibraryInfoImpl*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::StringRef, llvm::ArrayRef<llvm::PassPlugin>, llvm::ArrayRef<std::function<void (llvm::PassBuilder&)>>, llvm::opt_tool::OutputKind, llvm::opt_tool::VerifierKind, bool, bool, bool, bool, bool, bool, bool) (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x582ce5)
#26 0x00005f05c5b5f316 optMain (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x575316)
#27 0x000071a702429d90 __libc_start_call_main ./csu/../sysdeps/nptl/libc_start_call_main.h:58:16
#28 0x000071a702429e40 call_init ./csu/../csu/libc-start.c:128:20
#29 0x000071a702429e40 __libc_start_main ./csu/../csu/libc-start.c:379:5
#30 0x00005f05c5b54f95 _start (/scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt+0x56af95)
zsh: IOT instruction (core dumped)  /scratch/tc-testing/tc-apr-2/build-rv64gcv/build-llvm-linux/bin/opt
```
Godbolt: https://godbolt.org/z/bvY138WfG

Triggered assertion: https://github.com/llvm/llvm-project/blob/f5991161529511ca6ebc058da2a0507c2fc5283e/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp#L58

Found via fuzzer


commit 7506872afc6c1cc7556eed5128a2030b6eb8ca29
Author: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date:   Thu Oct 3 11:47:08 2024 +0100

    [DebugInfo][LSR] Fix assertion failure salvaging IV with offset > 64 bits wide (#110979)

    Fixes #110494

diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index f69db57c25ed..575395eda1c5 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -6870,6 +6870,8 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
             SE.computeConstantDifference(DVIRec.SCEVs[i], SCEVInductionVar)) {
       if (Offset->getSignificantBits() <= 64)
         SalvageExpr->createOffsetExpr(Offset->getSExtValue(), LSRInductionVar);
+      else
+        return false;
     } else if (!SalvageExpr->createIterCountExpr(DVIRec.SCEVs[i], IterCountExpr,
                                                  SE))
       return false;

define i16 @main() {
entry:
  br label %for.cond29

for.cond29:                                       ; preds = %for.body32, %entry
  %il_1000.0 = phi i128 [ 0, %entry ], [ %inc72, %for.body32 ]
  %l_995.0 = phi i128 [ 4704496199548239085565, %entry ], [ %inc70, %for.body32 ]
    #dbg_value(i128 %l_995.0, !4, !DIExpression(), !9)
  %cmp30 = icmp slt i128 %il_1000.0, 0
  br i1 %cmp30, label %for.body32, label %for.cond.cleanup31

for.cond.cleanup31:                               ; preds = %for.cond29
  ret i16 0

for.body32:                                       ; preds = %for.cond29
  %inc70 = add i128 %l_995.0, 1
  %inc72 = add i128 %il_1000.0, 1
  br label %for.cond29
}

!llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!3}

!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, globals: !2, splitDebugInlining: false, nameTableKind: None)
!1 = !DIFile(filename: "foo.c", directory: "/tmp")
!2 = !{}
!3 = !{i32 2, !"Debug Info Version", i32 3}
!4 = !DILocalVariable(name: "l_995", scope: !5, file: !1, line: 414, type: !7)
!5 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 397, type: !6, scopeLine: 398, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
!6 = !DISubroutineType(types: !2)
!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint128_t", file: !1, baseType: !8)
!8 = !DIBasicType(name: "unsigned __int128", size: 128, encoding: DW_ATE_unsigned)
!9 = !DILocation(line: 0, scope: !5)

llc: ../lib/Transforms/Scalar/LoopStrengthReduce.cpp:6630: [...] Assertion `!LocationOps.empty() && "Expected the location ops to contain the IV."' failed.

llvm commit: 82e594a022d
Reproduce with: ```llc bbi-99605.ll -o /dev/null```
Result:
```
llc: ../lib/Transforms/Scalar/LoopStrengthReduce.cpp:6630: void (anonymous namespace)::SCEVDbgValueBuilder::appendToVectors(SmallVectorImpl<uint64_t> &, SmallVectorImpl<llvm::Value *> &): Assertion `!LocationOps.empty() && "Expected the location ops to contain the IV."' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.	Program arguments: build-all/bin/llc bbi-99605.ll -o /dev/null
1.	Running pass 'Function Pass Manager' on module 'bbi-99605.ll'.
2.	Running pass 'Loop Pass Manager' on function '@main'
3.	Running pass 'Loop Strength Reduction' on basic block '%for.cond29'
 #0 0x000055e150599557 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (build-all/bin/llc+0x6dee557)
 #1 0x000055e1505970be llvm::sys::RunSignalHandlers() (build-all/bin/llc+0x6dec0be)
 #2 0x000055e150599bff SignalHandler(int) Signals.cpp:0:0
 #3 0x00007f66b410bcf0 __restore_rt (/lib64/libpthread.so.0+0x12cf0)
 #4 0x00007f66b1cc4acf raise (/lib64/libc.so.6+0x4eacf)
 #5 0x00007f66b1c97ea5 abort (/lib64/libc.so.6+0x21ea5)
 #6 0x00007f66b1c97d79 _nl_load_domain.cold.0 (/lib64/libc.so.6+0x21d79)
 #7 0x00007f66b1cbd426 (/lib64/libc.so.6+0x47426)
 #8 0x000055e15005db21 (build-all/bin/llc+0x68b2b21)
 #9 0x000055e15005c0c3 SalvageDVI(llvm::Loop*, llvm::ScalarEvolution&, llvm::PHINode*, (anonymous namespace)::DVIRecoveryRec&, llvm::SCEV const*, (anonymous namespace)::SCEVDbgValueBuilder) LoopStrengthReduce.cpp:0:0
#10 0x000055e15002dca8 ReduceLoopStrength(llvm::Loop*, llvm::IVUsers&, llvm::ScalarEvolution&, llvm::DominatorTree&, llvm::LoopInfo&, llvm::TargetTransformInfo const&, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::MemorySSA*) LoopStrengthReduce.cpp:0:0
#11 0x000055e15005e7e8 (anonymous namespace)::LoopStrengthReduce::runOnLoop(llvm::Loop*, llvm::LPPassManager&) LoopStrengthReduce.cpp:0:0
#12 0x000055e14f26f0f9 llvm::LPPassManager::runOnFunction(llvm::Function&) (build-all/bin/llc+0x5ac40f9)
#13 0x000055e14fba9e38 llvm::FPPassManager::runOnFunction(llvm::Function&) (build-all/bin/llc+0x63fee38)
#14 0x000055e14fbb2171 llvm::FPPassManager::runOnModule(llvm::Module&) (build-all/bin/llc+0x6407171)
#15 0x000055e14fbaa8b5 llvm::legacy::PassManagerImpl::run(llvm::Module&) (build-all/bin/llc+0x63ff8b5)
#16 0x000055e14d988be8 compileModule(char**, llvm::LLVMContext&) llc.cpp:0:0
#17 0x000055e14d98626d main (build-all/bin/llc+0x41db26d)
#18 0x00007f66b1cb0d85 __libc_start_main (/lib64/libc.so.6+0x3ad85)
#19 0x000055e14d9822ee _start (build-all/bin/llc+0x41d72ee)
Abort (core dumped)
```
The input contains the new debug info:
```#dbg_value(i128 %l_995.0, !4, !DIExpression(), !9)```

[bbi-99605.ll.gz](https://github.com/user-attachments/files/17189125/bbi-99605.ll.gz)

Thanks for the small reproducer. That is fixed in #110979 - this PR just avoids us crash/asserting when a salvage attempt is made with an offset over 64 bits wide and poisons the dbg record instead. I didn't look into whether we could theoretically or practically salvage it.

n.b., this case was broken with debug intrinsics too, not just the new debug record format.


commit 092d6283838dea79670750b9415955c5f0cb5178
Author: Alexey Bataev <a.bataev@outlook.com>
Date:   Mon Jan 13 09:02:56 2025 -0800

    [SLP]Check for div/rem instructions before extending with poisons

    Need to check if the instructions can be safely extended with poison
    before actually doing this to avoid incorrect transformations.

    Fixes #122691

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index df46c69ff3ab..4b0ed5b30179 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -8091,6 +8091,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
             NonUniqueValueVL.append(
                 PWSz - UniqueValues.size(),
                 PoisonValue::get(UniqueValues.front()->getType()));
+            // Check that extended with poisons operations are still valid for
+            // vectorization (div/rem are not allowed).
+            if (!getSameOpcode(NonUniqueValueVL, *TLI).valid()) {
+              LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+              newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+              return false;
+            }
             VL = NonUniqueValueVL;
           }
           return true;
@@ -17818,7 +17825,7 @@ bool BoUpSLP::collectValuesToDemote(
   };
   if (E.isGather() || !Visited.insert(&E).second ||
       any_of(E.Scalars, [&](Value *V) {
-        return all_of(V->users(), [&](User *U) {
+        return !isa<PoisonValue>(V) && all_of(V->users(), [&](User *U) {
           return isa<InsertElementInst>(U) && !getTreeEntry(U);
         });
       }))

[SLPVectorizer] Assertion `isa<To>(Val) && "cast<Ty>() argument of incompatible type!"' failed.

Reproducer: https://godbolt.org/z/68MKbnE4W
```
; bin/opt -O2 reduced.ll -S
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@g_3 = internal unnamed_addr global i64 0
@g_127 = external local_unnamed_addr global [3 x i32]
@g_211 = external local_unnamed_addr global i32

define i8 @func_125() {
entry:
  %call42 = call ptr @func_178()
  ret i8 0
}

define ptr @func_178() {
entry:
  %0 = load i32, ptr @g_211, align 4
  %tobool158.not = icmp eq i32 %0, 0
  br i1 %tobool158.not, label %for.inc434, label %for.cond166.preheader

for.cond166.preheader:                            ; preds = %entry
  br label %for.cond166

for.cond166:                                      ; preds = %for.cond166.preheader, %for.cond166
  %1 = phi i64 [ %2, %for.cond166 ], [ poison, %for.cond166.preheader ]
  %2 = phi i64 [ %inc255, %for.cond166 ], [ 0, %for.cond166.preheader ]
  %cmp167 = icmp samesign ult i64 %2, 61
  %inc255 = add nuw nsw i64 %2, 1
  br i1 %cmp167, label %for.cond166, label %for.inc434.loopexit

for.inc434.loopexit:                              ; preds = %for.cond166
  %.lcssa = phi i64 [ %1, %for.cond166 ]
  store i64 %.lcssa, ptr @g_3, align 8
  br label %for.inc434

for.inc434:                                       ; preds = %for.inc434.loopexit, %entry
  store i32 1, ptr @g_211, align 4
  %3 = load i64, ptr @g_3, align 8
  %conv.i = trunc nuw nsw i64 %3 to i32
  %4 = load i32, ptr @g_127, align 4
  %and.i = and i32 %4, %conv.i
  %cmp.i = icmp eq i32 %and.i, 0
  %conv8.i = sext i32 %and.i to i64
  %sext = shl i64 %3, 32
  %conv9.i = ashr exact i64 %sext, 32
  %g_211.promoted5 = load i32, ptr @g_211, align 4
  br label %for.body447

for.body447:                                      ; preds = %for.inc434, %cleanup552
  %p_181.addr.08 = phi i64 [ -13, %for.inc434 ], [ %conv564, %cleanup552 ]
  %and14.i1.lcssa67 = phi i32 [ %g_211.promoted5, %for.inc434 ], [ %and14.i.lcssa, %cleanup552 ]
  br label %if.then489

if.then489:                                       ; preds = %for.body447, %safe_div_func_int64_t_s_s.exit
  %l_317.03 = phi i32 [ 2, %for.body447 ], [ %sub, %safe_div_func_int64_t_s_s.exit ]
  %and14.i12 = phi i32 [ %and14.i1.lcssa67, %for.body447 ], [ %and14.i, %safe_div_func_int64_t_s_s.exit ]
  br i1 %cmp.i, label %safe_div_func_int64_t_s_s.exit, label %lor.lhs.false.i

lor.lhs.false.i:                                  ; preds = %if.then489
  %div.i = sdiv i64 %conv8.i, %conv9.i
  %5 = trunc i64 %div.i to i16
  br label %safe_div_func_int64_t_s_s.exit

safe_div_func_int64_t_s_s.exit:                   ; preds = %lor.lhs.false.i, %if.then489
  %cond.i = phi i16 [ %5, %lor.lhs.false.i ], [ 0, %if.then489 ]
  %call12.i = tail call i16 @llvm.bswap.i16(i16 %cond.i)
  %conv13.i = zext i16 %call12.i to i32
  %and14.i = and i32 %and14.i12, %conv13.i
  %call26.i = load volatile ptr, ptr null, align 8
  %sub = add nsw i32 %l_317.03, -1
  %cmp482 = icmp sgt i32 %l_317.03, 0
  br i1 %cmp482, label %if.then489, label %cleanup552

cleanup552:                                       ; preds = %safe_div_func_int64_t_s_s.exit
  %and14.i.lcssa = phi i32 [ %and14.i, %safe_div_func_int64_t_s_s.exit ]
  %add.i = add nsw i64 %p_181.addr.08, 7
  %conv564 = and i64 %add.i, 255
  %6 = and i64 %p_181.addr.08, 255
  %cmp445.not = icmp eq i64 %6, 22
  br i1 %cmp445.not, label %for.end565, label %for.body447

for.end565:                                       ; preds = %cleanup552
  %and14.i.lcssa.lcssa = phi i32 [ %and14.i.lcssa, %cleanup552 ]
  store i32 %conv.i, ptr getelementptr inbounds nuw (i8, ptr @g_127, i64 8), align 4
  store i32 %and14.i.lcssa.lcssa, ptr @g_211, align 4
  ret ptr null
}

define ptr @func_183(i64 %p_185) {
entry:
  %call = call i64 @builtin_uaddl_overflow(i64 %p_185, i64 0)
  unreachable
}

define i64 @builtin_uaddl_overflow(i64 %x, i64 %y) {
entry:
  %0 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %x, i64 %y)
  %1 = extractvalue { i64, i1 } %0, 1
  %conv = zext i1 %1 to i64
  ret i64 %conv
}
```
```
opt: /root/llvm-project/llvm/include/llvm/Support/Casting.h:578: decltype(auto) llvm::cast(From*) [with To = llvm::Instruction; From = llvm::Value]: Assertion `isa<To>(Val) && "cast<Ty>() argument of incompatible type!"' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.	Program arguments: /opt/compiler-explorer/clang-assertions-trunk/bin/opt -o /app/output.s -S -O2 <source>
1.	Running pass "function<eager-inv>(float2int,lower-constant-intrinsics,loop(loop-rotate<header-duplication;no-prepare-for-lto>,loop-deletion),loop-distribute,inject-tli-mappings,loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>,infer-alignment,loop-load-elim,instcombine<max-iterations=1;no-verify-fixpoint>,simplifycfg<bonus-inst-threshold=1;forward-switch-cond;switch-range-to-icmp;switch-to-lookup;no-keep-loops;hoist-common-insts;no-hoist-loads-stores-with-cond-faulting;sink-common-insts;speculate-blocks;simplify-cond-branch;no-speculate-unpredictables>,slp-vectorizer,vector-combine,instcombine<max-iterations=1;no-verify-fixpoint>,loop-unroll<O2>,transform-warning,sroa<preserve-cfg>,infer-alignment,instcombine<max-iterations=1;no-verify-fixpoint>,loop-mssa(licm<allowspeculation>),alignment-from-assumptions,loop-sink,instsimplify,div-rem-pairs,tailcallelim,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;hoist-loads-stores-with-cond-faulting;no-sink-common-insts;speculate-blocks;simplify-cond-branch;speculate-unpredictables>)" on module "<source>"
2.	Running pass "slp-vectorizer" on function "func_125"
 #0 0x00000000052803a8 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x52803a8)
 #1 0x000000000527ddac SignalHandler(int) Signals.cpp:0:0
 #2 0x0000776446c42520 (/lib/x86_64-linux-gnu/libc.so.6+0x42520)
 #3 0x0000776446c969fc pthread_kill (/lib/x86_64-linux-gnu/libc.so.6+0x969fc)
 #4 0x0000776446c42476 gsignal (/lib/x86_64-linux-gnu/libc.so.6+0x42476)
 #5 0x0000776446c287f3 abort (/lib/x86_64-linux-gnu/libc.so.6+0x287f3)
 #6 0x0000776446c2871b (/lib/x86_64-linux-gnu/libc.so.6+0x2871b)
 #7 0x0000776446c39e96 (/lib/x86_64-linux-gnu/libc.so.6+0x39e96)
 #8 0x0000000000b6a41a (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0xb6a41a)
 #9 0x0000000004253c98 llvm::slpvectorizer::BoUpSLP::collectValuesToDemote(llvm::slpvectorizer::BoUpSLP::TreeEntry const&, bool, unsigned int&, llvm::SmallVectorImpl<unsigned int>&, llvm::DenseSet<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, llvm::DenseMapInfo<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, void>>&, llvm::SmallDenseSet<unsigned int, 8u, llvm::DenseMapInfo<unsigned int, void>> const&, unsigned int&, bool&, bool) const::'lambda10'(unsigned int, unsigned int)::operator()(unsigned int, unsigned int) const::'lambda'(llvm::Value*)::operator()(llvm::Value*) const SLPVectorizer.cpp:0:0
#10 0x0000000004253f67 bool llvm::function_ref<bool (unsigned int, unsigned int)>::callback_fn<llvm::slpvectorizer::BoUpSLP::collectValuesToDemote(llvm::slpvectorizer::BoUpSLP::TreeEntry const&, bool, unsigned int&, llvm::SmallVectorImpl<unsigned int>&, llvm::DenseSet<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, llvm::DenseMapInfo<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, void>>&, llvm::SmallDenseSet<unsigned int, 8u, llvm::DenseMapInfo<unsigned int, void>> const&, unsigned int&, bool&, bool) const::'lambda10'(unsigned int, unsigned int)>(long, unsigned int, unsigned int) SLPVectorizer.cpp:0:0
#11 0x00000000042e2312 llvm::slpvectorizer::BoUpSLP::collectValuesToDemote(llvm::slpvectorizer::BoUpSLP::TreeEntry const&, bool, unsigned int&, llvm::SmallVectorImpl<unsigned int>&, llvm::DenseSet<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, llvm::DenseMapInfo<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, void>>&, llvm::SmallDenseSet<unsigned int, 8u, llvm::DenseMapInfo<unsigned int, void>> const&, unsigned int&, bool&, bool) const::'lambda6'(unsigned int&, llvm::ArrayRef<llvm::slpvectorizer::BoUpSLP::TreeEntry const*>, llvm::function_ref<bool (unsigned int, unsigned int)>)::operator()(unsigned int&, llvm::ArrayRef<llvm::slpvectorizer::BoUpSLP::TreeEntry const*>, llvm::function_ref<bool (unsigned int, unsigned int)>) const SLPVectorizer.cpp:0:0
#12 0x00000000042deacb llvm::slpvectorizer::BoUpSLP::collectValuesToDemote(llvm::slpvectorizer::BoUpSLP::TreeEntry const&, bool, unsigned int&, llvm::SmallVectorImpl<unsigned int>&, llvm::DenseSet<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, llvm::DenseMapInfo<llvm::slpvectorizer::BoUpSLP::TreeEntry const*, void>>&, llvm::SmallDenseSet<unsigned int, 8u, llvm::DenseMapInfo<unsigned int, void>> const&, unsigned int&, bool&, bool) const (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x42deacb)
#13 0x00000000042df8a0 llvm::slpvectorizer::BoUpSLP::computeMinimumValueSizes()::'lambda0'(llvm::slpvectorizer::BoUpSLP::TreeEntry const&, bool, bool, unsigned int, unsigned int, bool, bool)::operator()(llvm::slpvectorizer::BoUpSLP::TreeEntry const&, bool, bool, unsigned int, unsigned int, bool, bool) const SLPVectorizer.cpp:0:0
#14 0x00000000042e03d0 llvm::slpvectorizer::BoUpSLP::computeMinimumValueSizes() (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x42e03d0)
#15 0x000000000431e02d (anonymous namespace)::HorizontalReduction::tryToReduce(llvm::slpvectorizer::BoUpSLP&, llvm::DataLayout const&, llvm::TargetTransformInfo*, llvm::TargetLibraryInfo const&, llvm::AssumptionCache*) SLPVectorizer.cpp:0:0
#16 0x000000000432015d llvm::SLPVectorizerPass::vectorizeHorReduction(llvm::PHINode*, llvm::Instruction*, llvm::BasicBlock*, llvm::slpvectorizer::BoUpSLP&, llvm::SmallVectorImpl<llvm::WeakTrackingVH>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x432015d)
#17 0x000000000432494b llvm::SLPVectorizerPass::vectorizeRootInstruction(llvm::PHINode*, llvm::Instruction*, llvm::BasicBlock*, llvm::slpvectorizer::BoUpSLP&) (.constprop.0) SLPVectorizer.cpp:0:0
#18 0x000000000432904f llvm::SLPVectorizerPass::vectorizeChainsInBlock(llvm::BasicBlock*, llvm::slpvectorizer::BoUpSLP&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x432904f)
#19 0x000000000432fa86 llvm::SLPVectorizerPass::runImpl(llvm::Function&, llvm::ScalarEvolution*, llvm::TargetTransformInfo*, llvm::TargetLibraryInfo*, llvm::AAResults*, llvm::LoopInfo*, llvm::DominatorTree*, llvm::AssumptionCache*, llvm::DemandedBits*, llvm::OptimizationRemarkEmitter*) (.part.0) SLPVectorizer.cpp:0:0
#20 0x00000000043305bb llvm::SLPVectorizerPass::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x43305bb)
#21 0x0000000003036f1e llvm::detail::PassModel<llvm::Function, llvm::SLPVectorizerPass, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x3036f1e)
#22 0x000000000507d7f0 llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x507d7f0)
#23 0x0000000000e7023e llvm::detail::PassModel<llvm::Function, llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0xe7023e)
#24 0x000000000507c10b llvm::ModuleToFunctionPassAdaptor::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x507c10b)
#25 0x0000000000e6f8ce llvm::detail::PassModel<llvm::Module, llvm::ModuleToFunctionPassAdaptor, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0xe6f8ce)
#26 0x000000000507bb10 llvm::PassManager<llvm::Module, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x507bb10)
#27 0x000000000094448a llvm::runPassPipeline(llvm::StringRef, llvm::Module&, llvm::TargetMachine*, llvm::TargetLibraryInfoImpl*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::StringRef, llvm::ArrayRef<llvm::PassPlugin>, llvm::ArrayRef<std::function<void (llvm::PassBuilder&)>>, llvm::opt_tool::OutputKind, llvm::opt_tool::VerifierKind, bool, bool, bool, bool, bool, bool, bool) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x94448a)
#28 0x0000000000936a29 optMain (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x936a29)
#29 0x0000776446c29d90 (/lib/x86_64-linux-gnu/libc.so.6+0x29d90)
#30 0x0000776446c29e40 __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29e40)
#31 0x000000000092e305 _start (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x92e305)
Program terminated with signal: SIGSEGV
Compiler returned: 139
```

llvm version: 7e01a322f850e86be9eefde8ae5a30e532d22cfa
I am working to provide a single-pass reproducer.

Missing poison check: https://github.com/llvm/llvm-project/blob/6c5941b09fca487efc5000c82bbce6054bf36a7c/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp#L18000

> I am working to provide a single-pass reproducer.

It is weird that the single-pass reproducer doesn't work even with `--preserve-ll-uselistorder`.


commit d1a7225076218ce224cd29c74259b715b393dc9d
Author: Alexey Bataev <a.bataev@outlook.com>
Date:   Mon Dec 16 07:52:32 2024 -0800

    [SLP]Check if the node must keep its original bitwidth

    Need to check if during previous analysis the node has requested to keep
    its original bitwidth to avoid incorrect codegen.

    Fixes #120076

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3bd983ee6e12..d967813075bb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2931,13 +2931,11 @@ private:
   /// truncation. We collect the entries that will be demoted in ToDemote.
   /// \param E Node for analysis
   /// \param ToDemote indices of the nodes to be demoted.
-  bool collectValuesToDemote(const TreeEntry &E, bool IsProfitableToDemoteRoot,
-                             unsigned &BitWidth,
-                             SmallVectorImpl<unsigned> &ToDemote,
-                             DenseSet<const TreeEntry *> &Visited,
-                             unsigned &MaxDepthLevel,
-                             bool &IsProfitableToDemote,
-                             bool IsTruncRoot) const;
+  bool collectValuesToDemote(
+      const TreeEntry &E, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
+      SmallVectorImpl<unsigned> &ToDemote, DenseSet<const TreeEntry *> &Visited,
+      const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
+      bool &IsProfitableToDemote, bool IsTruncRoot) const;

   /// Check if the operands on the edges \p Edges of the \p UserTE allows
   /// reordering (i.e. the operands can be reordered because they have only one
@@ -17515,8 +17513,8 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) {
 bool BoUpSLP::collectValuesToDemote(
     const TreeEntry &E, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
     SmallVectorImpl<unsigned> &ToDemote, DenseSet<const TreeEntry *> &Visited,
-    unsigned &MaxDepthLevel, bool &IsProfitableToDemote,
-    bool IsTruncRoot) const {
+    const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
+    bool &IsProfitableToDemote, bool IsTruncRoot) const {
   // We can always demote constants.
   if (all_of(E.Scalars, IsaPred<Constant>))
     return true;
@@ -17528,6 +17526,10 @@ bool BoUpSLP::collectValuesToDemote(
     return true;
   }

+  // Check if the node was analyzed already and must keep its original bitwidth.
+  if (NodesToKeepBWs.contains(E.Idx))
+    return false;
+
   // If the value is not a vectorized instruction in the expression and not used
   // by the insertelement instruction and not used in multiple vector nodes, it
   // cannot be demoted.
@@ -17623,8 +17625,8 @@ bool BoUpSLP::collectValuesToDemote(
     for (const TreeEntry *Op : Operands) {
       unsigned Level = InitLevel;
       if (!collectValuesToDemote(*Op, IsProfitableToDemoteRoot, BitWidth,
-                                 ToDemote, Visited, Level, IsProfitableToDemote,
-                                 IsTruncRoot)) {
+                                 ToDemote, Visited, NodesToKeepBWs, Level,
+                                 IsProfitableToDemote, IsTruncRoot)) {
         if (!IsProfitableToDemote)
           return false;
         NeedToExit = true;
@@ -17926,6 +17928,7 @@ void BoUpSLP::computeMinimumValueSizes() {
   bool IsTruncRoot = false;
   bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
   SmallVector<unsigned> RootDemotes;
+  SmallDenseSet<unsigned, 8> NodesToKeepBWs;
   if (NodeIdx != 0 &&
       VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
       VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
@@ -17949,6 +17952,7 @@ void BoUpSLP::computeMinimumValueSizes() {
     // Check if the root is trunc and the next node is gather/buildvector, then
     // keep trunc in scalars, which is free in most cases.
     if (E.isGather() && IsTruncRoot && E.UserTreeIndices.size() == 1 &&
+        !NodesToKeepBWs.contains(E.Idx) &&
         E.Idx > (IsStoreOrInsertElt ? 2u : 1u) &&
         all_of(E.Scalars, [&](Value *V) {
           return V->hasOneUse() || isa<Constant>(V) ||
@@ -18071,8 +18075,8 @@ void BoUpSLP::computeMinimumValueSizes() {
     bool NeedToDemote = IsProfitableToDemote;

     if (!collectValuesToDemote(E, IsProfitableToDemoteRoot, MaxBitWidth,
-                               ToDemote, Visited, MaxDepthLevel, NeedToDemote,
-                               IsTruncRoot) ||
+                               ToDemote, Visited, NodesToKeepBWs, MaxDepthLevel,
+                               NeedToDemote, IsTruncRoot) ||
         (MaxDepthLevel <= Limit &&
          !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
             (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
@@ -18206,7 +18210,7 @@ void BoUpSLP::computeMinimumValueSizes() {
                  });
     }

-    // If the maximum bit width we compute is less than the with of the roots'
+    // If the maximum bit width we compute is less than the width of the roots'
     // type, we can proceed with the narrowing. Otherwise, do nothing.
     if (MaxBitWidth == 0 ||
         MaxBitWidth >=
@@ -18214,6 +18218,7 @@ void BoUpSLP::computeMinimumValueSizes() {
                 ->getBitWidth()) {
       if (UserIgnoreList)
         AnalyzedMinBWVals.insert(TreeRoot.begin(), TreeRoot.end());
+      NodesToKeepBWs.insert(ToDemote.begin(), ToDemote.end());
       continue;
     }

opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s

define i8 @test() {
entry:
  %conv4.i.i = zext i16 0 to i32
  %conv7.i.i = sext i16 0 to i32
  %cmp8.i.i = icmp slt i32 %conv7.i.i, %conv4.i.i
  %conv9.i.i = zext i1 %cmp8.i.i to i32
  %or10.i.i = or i32 %conv9.i.i, %conv4.i.i
  %cmp11.i.i = icmp eq i32 %or10.i.i, %conv4.i.i
  %sub.i.i79.peel.i = sub i16 0, 1
  %xor5.i81.peel.i = zext i16 %sub.i.i79.peel.i to i32
  %conv7.i84.peel.i = sext i16 0 to i32
  %cmp8.i85.peel.i = icmp slt i32 %conv7.i84.peel.i, %xor5.i81.peel.i
  %conv9.i86.peel.i = zext i1 %cmp8.i85.peel.i to i32
  %or10.i87.peel.i = or i32 %conv9.i86.peel.i, %xor5.i81.peel.i
  %cmp11.i88.peel.i = icmp eq i32 %or10.i87.peel.i, %xor5.i81.peel.i
  %conv13.i89.peel.i = zext i1 %cmp8.i85.peel.i to i8
  ret i8 %conv13.i89.peel.i
}

[SLPVectorizer] Miscompile at O3

Reproducer: https://alive2.llvm.org/ce/z/_88AAB
```
; bin/opt -passes=slp-vectorizer reduced.ll -S
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define i8 @func_129() {
entry:
  %conv4.i.i = zext i16 0 to i32
  %conv7.i.i = sext i16 0 to i32
  %cmp8.i.i = icmp slt i32 %conv7.i.i, %conv4.i.i
  %conv9.i.i = zext i1 %cmp8.i.i to i32
  %or10.i.i = or i32 %conv9.i.i, %conv4.i.i
  %cmp11.i.i = icmp eq i32 %or10.i.i, %conv4.i.i
  %sub.i.i79.peel.i = sub i16 0, 1
  %xor5.i81.peel.i = zext i16 %sub.i.i79.peel.i to i32
  %conv7.i84.peel.i = sext i16 0 to i32
  %cmp8.i85.peel.i = icmp slt i32 %conv7.i84.peel.i, %xor5.i81.peel.i
  %conv9.i86.peel.i = zext i1 %cmp8.i85.peel.i to i32
  %or10.i87.peel.i = or i32 %conv9.i86.peel.i, %xor5.i81.peel.i
  %cmp11.i88.peel.i = icmp eq i32 %or10.i87.peel.i, %xor5.i81.peel.i
  %conv13.i89.peel.i = zext i1 %cmp8.i85.peel.i to i8
  ret i8 %conv13.i89.peel.i
}
```
```
define i8 @func_129() {
entry:
  %sub.i.i79.peel.i = sub i16 0, 1
  %0 = insertelement <2 x i16> <i16 poison, i16 0>, i16 %sub.i.i79.peel.i, i32 0
  %1 = icmp slt <2 x i16> zeroinitializer, %0
  %2 = zext <2 x i1> %1 to <2 x i16>
  %3 = or <2 x i16> %2, %0
  %4 = icmp eq <2 x i16> %3, %0
  %5 = extractelement <2 x i1> %1, i32 0
  %conv13.i89.peel.i = zext i1 %5 to i8
  ret i8 %conv13.i89.peel.i
}
```
```
----------------------------------------
define i8 @src() {
entry:
  %sub.i.i79.peel.i = sub i16 0, 1
  %xor5.i81.peel.i = zext i16 %sub.i.i79.peel.i to i32
  %conv7.i84.peel.i = sext i16 0 to i32
  %cmp8.i85.peel.i = icmp slt i32 %conv7.i84.peel.i, %xor5.i81.peel.i
  %conv13.i89.peel.i = zext i1 %cmp8.i85.peel.i to i8
  ret i8 %conv13.i89.peel.i
}
=>
define i8 @tgt() {
entry:
  %sub.i.i79.peel.i = sub i16 0, 1
  %#0 = insertelement <2 x i16> { poison, 0 }, i16 %sub.i.i79.peel.i, i32 0
  %#1 = icmp slt <2 x i16> { 0, 0 }, %#0
  %#5 = extractelement <2 x i1> %#1, i32 0
  %conv13.i89.peel.i = zext i1 %#5 to i8
  ret i8 %conv13.i89.peel.i
}
Transformation doesn't verify!

ERROR: Value mismatch

Example:

Source:
i16 %sub.i.i79.peel.i = #xffff (65535, -1)
i32 %xor5.i81.peel.i = #x0000ffff (65535)
i32 %conv7.i84.peel.i = #x00000000 (0)
i1 %cmp8.i85.peel.i = #x1 (1)
i8 %conv13.i89.peel.i = #x01 (1)

Target:
i16 %sub.i.i79.peel.i = #xffff (65535, -1)
<2 x i16> %#0 = < #xffff (65535, -1), #x0000 (0) >
<2 x i1> %#1 = < #x0 (0), #x0 (0) >
i1 %#5 = #x0 (0)
i8 %conv13.i89.peel.i = #x00 (0)
Source value: #x01 (1)
Target value: #x00 (0)
```

cc @alexey-bataev


commit 1de3dc7d23dd6b856efad3a3a04f2396328726d7
Author: Florian Hahn <flo@fhahn.com>
Date:   Tue Jan 14 22:07:38 2025 +0000

    [LV] Bail out early if BTC+1 wraps.

    Currently we fail to detect the case where BTC + 1 wraps, i.e. the
    vector trip count is 0, In those cases, the minimum iteration count
    check will fail, and the vector code will never be executed.

    Explicitly check for this condition in computeMaxVF and avoid trying to
    vectorize alltogether.

    Note that a number of tests needed to be updated, because the vector
    loop would never be executed given the input IR.

    Fixes https://github.com/llvm/llvm-project/issues/122558.

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 744faef19243..fe2fb5e9faae 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4052,7 +4052,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     return FixedScalableVFPair::getNone();
   }

-  unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
+  ScalarEvolution *SE = PSE.getSE();
+  unsigned TC = SE->getSmallConstantTripCount(TheLoop);
   unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
   LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
   if (TC != MaxTC)
@@ -4064,6 +4065,22 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     return FixedScalableVFPair::getNone();
   }

+  // If BTC matches the widest induction type and is -1 then the trip count
+  // computation will wrap to 0 and the vector trip count will be 0. Do not try
+  // to vectorize.
+  const SCEV *BTC = SE->getBackedgeTakenCount(TheLoop);
+  if (!isa<SCEVCouldNotCompute>(BTC) &&
+      BTC->getType()->getScalarSizeInBits() >=
+          Legal->getWidestInductionType()->getScalarSizeInBits() &&
+      SE->isKnownPredicate(CmpInst::ICMP_EQ, BTC,
+                           SE->getMinusOne(BTC->getType()))) {
+    reportVectorizationFailure(
+        "Trip count computation wrapped",
+        "backedge-taken count is -1, loop trip count wrapped to 0",
+        "TripCountWrapped", ORE, TheLoop);
+    return FixedScalableVFPair::getNone();
+  }
+
   switch (ScalarEpilogueStatus) {
   case CM_ScalarEpilogueAllowed:
     return computeFeasibleMaxVF(MaxTC, UserVF, false);

opt -mattr=+dotprod -passes=loop-vectorize -force-vector-interleave=1 -S < %s

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+i8mm,+dotprod -S < %s

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+dotprod -S < %s

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp_z_s(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp_s_z(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon,+dotprod -force-vector-interleave=1 -S < %s

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon,+dotprod -S < %s

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon,+dotprod -force-vector-interleave=1 -vectorizer-maximize-bandwidth -S < %s

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum3 = phi i32 [ 0, %entry ], [ %add.a3, %for.body ]
  %accum2 = phi i32 [ 0, %entry ], [ %add.a2, %for.body ]
  %accum1 = phi i32 [ 0, %entry ], [ %add.a1, %for.body ]
  %accum0 = phi i32 [ 0, %entry ], [ %add.a0, %for.body ]
  %gep.a0 = getelementptr inbounds i8, ptr %a, i64 %iv
  %gep.b0 = getelementptr inbounds i8, ptr %b, i64 %iv
  %offset.1 = or disjoint i64 %iv, 1
  %gep.a1 = getelementptr inbounds i8, ptr %a, i64 %offset.1
  %gep.b1 = getelementptr inbounds i8, ptr %b, i64 %offset.1
  %offset.2 = or disjoint i64 %iv, 2
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %offset.2
  %gep.b2 = getelementptr inbounds i8, ptr %b, i64 %offset.2
  %offset.3 = or disjoint i64 %iv, 3
  %gep.a3 = getelementptr inbounds i8, ptr %a, i64 %offset.3
  %gep.b3 = getelementptr inbounds i8, ptr %b, i64 %offset.3
  %load.a0 = load i8, ptr %gep.a0, align 1
  %ext.a0 = sext i8 %load.a0 to i32
  %load.b0 = load i8, ptr %gep.b0, align 1
  %ext.b0 = sext i8 %load.b0 to i32
  %mul.a0 = mul nsw i32 %ext.b0, %ext.a0
  %add.a0 = add nsw i32 %mul.a0, %accum0
  %load.a1 = load i8, ptr %gep.a1, align 1
  %ext.a1 = sext i8 %load.a1 to i32
  %load.b1 = load i8, ptr %gep.b1, align 1
  %ext.b1 = sext i8 %load.b1 to i32
  %mul.a1 = mul nsw i32 %ext.a1, %ext.b1
  %add.a1 = add nsw i32 %mul.a1, %accum1
  %load.a2 = load i8, ptr %gep.a2, align 1
  %ext.a2 = sext i8 %load.a2 to i32
  %load.b2 = load i8, ptr %gep.b2, align 1
  %ext.b2 = sext i8 %load.b2 to i32
  %mul.a2 = mul nsw i32 %ext.a2, %ext.b2
  %add.a2 = add nsw i32 %mul.a2, %accum2
  %load.a3 = load i8, ptr %gep.a3, align 1
  %ext.a3 = sext i8 %load.a3 to i32
  %load.b3 = load i8, ptr %gep.b3, align 1
  %ext.b3 = sext i8 %load.b3 to i32
  %mul.a3 = mul nsw i32 %ext.a3, %ext.b3
  %add.a3 = add nsw i32 %mul.a3, %accum3
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %num_in
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  %result0 = add nsw i32 %add.a0, %add.a1
  %result1 = add nsw i32 %add.a2, %add.a3
  %result = add nsw i32 %result0, %result1
  ret i32 %result
}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @dotp(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.b = load i8, ptr %gep.a2, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !0

exit:                                             ; preds = %for.body
  ret i32 %add
}

!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %mul, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_different_types(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i16, ptr %gep.b, align 2
  %ext.b = zext i16 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_not_phi(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %ext.b
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  %result = add i32 %add, %ext.b
  ret i32 %result
}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_predicated(i64 %N, ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i32 %add
}

opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s

opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define void @not_dotp_not_phi2(ptr %matrix, i32 %n) #0 {
entry:
  %cmp = icmp sgt i32 %n, 0
  br i1 %cmp, label %for.preheader, label %exit

for.preheader:                                    ; preds = %entry
  %load.a = load i8, ptr null, align 1
  %load.a1 = load i8, ptr inttoptr (i64 1 to ptr), align 1
  %a.ext = sext i8 %load.a to i32
  %a.ext1 = sext i8 %load.a1 to i32
  br label %for.body

for.body:                                         ; preds = %for.body, %for.preheader
  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %for.preheader ]
  %ptr = phi ptr [ %scevgep, %for.body ], [ %matrix, %for.preheader ]
  %accum = phi i32 [ %add.1, %for.body ], [ 0, %for.preheader ]
  %gep.b = getelementptr i8, ptr %ptr, i64 1
  %gep.b1 = getelementptr i8, ptr %ptr, i64 2
  %load.b = load i8, ptr %gep.b, align 1
  %b.ext = sext i8 %load.b to i32
  %mul = mul nsw i32 %a.ext, %b.ext
  %add = add i32 %mul, %accum
  %load.b1 = load i8, ptr %gep.b1, align 1
  %b.ext1 = sext i8 %load.b1 to i32
  %mul.1 = mul nsw i32 %a.ext1, %b.ext1
  %add.1 = add i32 %mul.1, %add
  %scevgep = getelementptr i8, ptr %ptr, i64 16
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond.not = icmp eq i32 %iv.next, %n
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  %add.1.lcssa = phi i32 [ %add.1, %for.body ]
  %add.float = sitofp i32 %add.1.lcssa to float
  br label %exit

exit:                                             ; preds = %for.exit, %entry
  %result = phi float [ 0.000000e+00, %entry ], [ %add.float, %for.exit ]
  store float %result, ptr %matrix, align 4
  ret void
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
entry:
  %cmp = icmp eq i64 %n, 0
  br i1 %cmp, label %exit, label %for.ph

for.ph:                                           ; preds = %entry
  %ext.b = zext i16 %b to i64
  br label %for.body

for.body:                                         ; preds = %for.body, %for.ph
  %iv = phi i64 [ 0, %for.ph ], [ %iv.next, %for.body ]
  %accum = phi i64 [ 0, %for.ph ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds nuw i16, ptr %a, i64 %iv
  %load.a = load i16, ptr %gep.a, align 2
  %ext.a = zext i16 %load.a to i64
  %mul = mul nuw nsw i64 %ext.b, %ext.a
  %add = add i64 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %cmp.1 = icmp eq i64 %iv.next, %n
  br i1 %cmp.1, label %exit, label %for.body

exit:                                             ; preds = %for.body, %entry
  %result = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum3 = phi i32 [ 0, %entry ], [ %add.a3, %for.body ]
  %accum2 = phi i32 [ 0, %entry ], [ %add.a2, %for.body ]
  %accum1 = phi i32 [ 0, %entry ], [ %add.a1, %for.body ]
  %accum0 = phi i32 [ 0, %entry ], [ %add.a0, %for.body ]
  %gep.a0 = getelementptr inbounds i8, ptr %a, i64 %iv
  %gep.b0 = getelementptr inbounds i8, ptr %b, i64 %iv
  %offset.1 = or disjoint i64 %iv, 1
  %gep.a1 = getelementptr inbounds i8, ptr %a, i64 %offset.1
  %gep.b1 = getelementptr inbounds i8, ptr %b, i64 %offset.1
  %offset.2 = or disjoint i64 %iv, 2
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %offset.2
  %gep.b2 = getelementptr inbounds i8, ptr %b, i64 %offset.2
  %offset.3 = or disjoint i64 %iv, 3
  %gep.a3 = getelementptr inbounds i8, ptr %a, i64 %offset.3
  %gep.b3 = getelementptr inbounds i8, ptr %b, i64 %offset.3
  %load.a0 = load i8, ptr %gep.a0, align 1
  %ext.a0 = sext i8 %load.a0 to i32
  %load.b0 = load i8, ptr %gep.b0, align 1
  %ext.b0 = sext i8 %load.b0 to i32
  %mul.a0 = mul nsw i32 %ext.b0, %ext.a0
  %add.a0 = add nsw i32 %mul.a0, %accum0
  %load.a1 = load i8, ptr %gep.a1, align 1
  %ext.a1 = sext i8 %load.a1 to i32
  %load.b1 = load i8, ptr %gep.b1, align 1
  %ext.b1 = sext i8 %load.b1 to i32
  %mul.a1 = mul nsw i32 %ext.a1, %ext.b1
  %add.a1 = add nsw i32 %mul.a1, %accum1
  %load.a2 = load i8, ptr %gep.a2, align 1
  %ext.a2 = sext i8 %load.a2 to i32
  %load.b2 = load i8, ptr %gep.b2, align 1
  %ext.b2 = sext i8 %load.b2 to i32
  %mul.a2 = mul nsw i32 %ext.a2, %ext.b2
  %add.a2 = add nsw i32 %mul.a2, %accum2
  %load.a3 = load i8, ptr %gep.a3, align 1
  %ext.a3 = sext i8 %load.a3 to i32
  %load.b3 = load i8, ptr %gep.b3, align 1
  %ext.b3 = sext i8 %load.b3 to i32
  %mul.a3 = mul nsw i32 %ext.a3, %ext.b3
  %add.a3 = add nsw i32 %mul.a3, %accum3
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %num_in
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  %result0 = add nsw i32 %add.a0, %add.a1
  %result1 = add nsw i32 %add.a2, %add.a3
  %result = add nsw i32 %result0, %result1
  ret i32 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.b = load i8, ptr %gep.a2, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !0

exit:                                             ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
  %sum = phi i64 [ 0, %entry ], [ %add, %for.body ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.iv
  %0 = load i8, ptr %arrayidx, align 1
  %conv = zext i8 %0 to i64
  %i.iv.next = add nuw nsw i64 %i.iv, 1
  %arrayidx2 = getelementptr inbounds nuw i8, ptr %b, i64 %i.iv.next
  %1 = load i8, ptr %arrayidx2, align 1
  %conv3 = zext i8 %1 to i64
  %mul = mul nuw nsw i64 %conv3, %conv
  %add = add i64 %sum, %mul
  %exitcond.not = icmp eq i64 %i.iv.next, 16
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i64 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %mul, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i16, ptr %gep.b, align 2
  %ext.b = zext i16 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %ext.b
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  %result = add i32 %add, %ext.b
  ret i32 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
entry:
  %cmp = icmp eq i64 %n, 0
  br i1 %cmp, label %exit, label %for.ph

for.ph:                                           ; preds = %entry
  %ext.b = zext i16 %b to i64
  br label %for.body

for.body:                                         ; preds = %for.body, %for.ph
  %iv = phi i64 [ 0, %for.ph ], [ %iv.next, %for.body ]
  %accum = phi i64 [ 0, %for.ph ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds nuw i16, ptr %a, i64 %iv
  %load.a = load i16, ptr %gep.a, align 2
  %ext.a = zext i16 %load.a to i64
  %mul = mul nuw nsw i64 %ext.a, %ext.b
  %add = add i64 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %cmp.1 = icmp eq i64 %iv.next, %n
  br i1 %cmp.1, label %exit, label %for.body

exit:                                             ; preds = %for.body, %entry
  %result = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon -S < %s

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1000
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

opt -p loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s

target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64-apple-macosx14.0.0"

; Function Attrs: vscale_range(1,16)
define void @trunc_store(ptr %dst, ptr %src, i16 %x) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %x.ext = zext i16 %x to i64
  %l = load i64, ptr %src, align 8
  %and = and i64 %l, %x.ext
  %trunc = trunc i64 %and to i8
  %gep = getelementptr i8, ptr %dst, i64 %iv
  store i8 %trunc, ptr %gep, align 1
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1000
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

opt -mattr=+neon,+dotprod -passes=loop-vectorize -debug-only=loop-vectorize -force-vector-interleave=1 -disable-output %s 2>&1

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @print_partial_reduction(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i32 %add
}

opt -mtriple=hexagon -hexagon-autohvx -passes=loop-vectorize -S < %s 2>&1

target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"

; Function Attrs: optsize
define i32 @f(ptr %src) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %g.016 = phi i32 [ 0, %entry ], [ %g.1.lcssa, %loop ]
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %0 = load i8, ptr %src, align 1
  %g.1.lcssa = add i32 %g.016, 1
  %iv.next = add nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, 1000
  br i1 %exitcond, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i32 %g.1.lcssa
}

attributes #0 = { optsize "target-features"="+hvx-length128b" }

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define i64 @second_lshr_operand_zero_via_scev() {
entry:
  %ext.0 = sext i8 0 to i32
  br label %loops

loops:                                            ; preds = %loops, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loops ]
  %red = phi i64 [ 0, %entry ], [ %red.next, %loops ]
  %c = icmp eq i64 %iv, 0
  %and = and i64 %iv, 0
  %0 = trunc i64 %iv to i32
  %shr = lshr i32 %0, %ext.0
  %conv.1 = zext i32 %shr to i64
  %red.next.v = select i1 %c, i64 %and, i64 %conv.1
  %red.next = or i64 %red.next.v, %red
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1000
  br i1 %ec, label %exit, label %loops

exit:                                             ; preds = %loops
  %res = phi i64 [ %red.next, %loops ]
  ret i64 %res
}

opt < %s -passes=loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S

target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"

define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
entry:
  %conv = zext i1 %cmp to i32
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 1, %entry ], [ %inc, %loop ]
  %or13 = phi i32 [ 0, %entry ], [ %or, %loop ]
  %and = and i32 %or13, 1
  %or = or i32 %and, %conv
  %inc = add i32 %iv, 1
  %ec = icmp eq i32 %iv, 16
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i32 %or
}

attributes #0 = { "target-cpu"="penryn" }

opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-pc-linux-gnu"

; Function Attrs: noinline nounwind uwtable
define void @drop_zext_nneg(ptr noalias %p, ptr noalias %p1) #0 {
entry:
  br label %body

body:                                             ; preds = %else, %entry
  %iv = phi i64 [ %next, %else ], [ 0, %entry ]
  %0 = trunc i64 %iv to i32
  %c = icmp eq i32 %0, 0
  br i1 %c, label %then, label %else

then:                                             ; preds = %body
  %zext = zext nneg i32 %0 to i64
  %idx1 = getelementptr double, ptr %p, i64 %zext
  %idx2 = getelementptr double, ptr %p, i64 %zext
  %1 = load double, ptr %idx2, align 8
  br label %else

else:                                             ; preds = %then, %body
  %phi = phi double [ %1, %then ], [ 0.000000e+00, %body ]
  store double %phi, ptr %p1, align 8
  %next = add i64 %iv, 1
  %cmp = icmp eq i64 %next, 1024
  br i1 %cmp, label %exit, label %body

exit:                                             ; preds = %else
  ret void
}

attributes #0 = { noinline nounwind uwtable "target-features"="+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl" }

target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.15.0"

define void @test_free_instructions_feeding_geps_for_interleave_groups(ptr noalias %p.invar, ptr noalias %dst.1, ptr noalias %dst.2) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %l.0 = load float, ptr %p.invar, align 4
  %iv.mul = shl i64 %iv, 2
  %gep.dst.19 = getelementptr float, ptr %dst.1, i64 %iv.mul
  store float %l.0, ptr %gep.dst.19, align 4
  %l.1 = load float, ptr %p.invar, align 4
  %add.1 = or disjoint i64 %iv.mul, 1
  %gep.dst.119 = getelementptr float, ptr %dst.1, i64 %add.1
  store float %l.1, ptr %gep.dst.119, align 4
  %add.2 = or disjoint i64 %iv.mul, 2
  %gep.dst.129 = getelementptr float, ptr %dst.1, i64 %add.2
  store float 0.000000e+00, ptr %gep.dst.129, align 4
  %add.3 = or disjoint i64 %iv.mul, 3
  %gep.dst.140 = getelementptr float, ptr %dst.1, i64 %add.3
  store float 0.000000e+00, ptr %gep.dst.140, align 4
  %l.2 = load float, ptr %p.invar, align 4
  %gep.dst.247 = getelementptr float, ptr %dst.2, i64 %iv.mul
  store float %l.2, ptr %gep.dst.247, align 4
  %gep.dst.255 = getelementptr float, ptr %dst.2, i64 %add.1
  store float 0.000000e+00, ptr %gep.dst.255, align 4
  %gep.dst.265 = getelementptr float, ptr %dst.2, i64 %add.2
  store float 0.000000e+00, ptr %gep.dst.265, align 4
  %gep.dst.276 = getelementptr float, ptr %dst.2, i64 %add.3
  store float 0.000000e+00, ptr %gep.dst.276, align 4
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define void @smax_call_uniform(ptr %dst, i64 %x) {
entry:
  %c = icmp ult i8 -68, -69
  %mul = mul nuw nsw i64 %x, 0
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 %c, label %loop.latch, label %else

else:                                             ; preds = %loop.header
  %rem = urem i64 %mul, %x
  %smax = tail call i64 @llvm.smax.i64(i64 %rem, i64 0)
  br label %loop.latch

loop.latch:                                       ; preds = %else, %loop.header
  %p = phi i64 [ 1, %loop.header ], [ %smax, %else ]
  %add = add i64 %p, 1
  %gep = getelementptr i64, ptr %dst, i64 %add
  store i64 0, ptr %gep, align 8
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

exit:                                             ; preds = %loop.latch
  ret void
}

; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
declare i64 @llvm.smax.i64(i64, i64) #0

attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }

opt -p loop-vectorize -force-vector-width=2 -epilogue-vectorization-force-VF=2 -S %s

target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"

define void @vector_trip_count_0_as_btc_is_all_1(ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.next = add i32 %iv, -1
  %gep = getelementptr inbounds i32, ptr %dst, i32 %iv
  store i32 %iv.next, ptr %gep, align 4
  %ec = icmp eq i32 %iv.next, 0
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s

define double @test_resinking_required(ptr %p, ptr noalias %a, ptr noalias %b) {
Entry:
  br label %Loop

Loop:                                             ; preds = %Loop, %Entry
  %for.1 = phi double [ %l1, %Loop ], [ 0.000000e+00, %Entry ]
  %for.2 = phi double [ %l2, %Loop ], [ 0.000000e+00, %Entry ]
  %for.3 = phi double [ %for.2, %Loop ], [ 0.000000e+00, %Entry ]
  %iv = phi i64 [ %iv.next, %Loop ], [ 0, %Entry ]
  %USE_2_FORS = fdiv double %for.3, %for.1
  %div = fdiv double 0.000000e+00, %for.1
  %l1 = load double, ptr %a, align 8
  %iv.next = add nuw nsw i64 %iv, 1
  %l2 = load double, ptr %b, align 8
  store double %div, ptr %p, align 8
  %cond = icmp eq i64 %iv.next, 1000
  br i1 %cond, label %End, label %Loop

End:                                              ; preds = %Loop
  %res.1 = fadd double %for.1, %for.2
  %res.2 = fadd double %res.1, %for.3
  ret double %res.2
}

opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 < %s

opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize -verify-loop-info < %s

opt -S -vectorize-num-stores-pred=1 -force-vector-width=2 -force-vector-interleave=1 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 < %s

target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"

define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.inc, %entry
  %tmp0 = phi i64 [ %tmp6, %for.inc ], [ 0, %entry ]
  %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 1000, %entry ]
  %tmp2 = getelementptr i8, ptr %ptr, i64 %tmp0
  %tmp3 = load i8, ptr %tmp2, align 1
  store i8 0, ptr %tmp2, align 1
  br i1 %c, label %if.then, label %for.inc

if.then:                                          ; preds = %for.body
  %tmp4 = zext i8 %tmp3 to i32
  %tmp5 = trunc i32 %tmp4 to i8
  store i8 %tmp5, ptr %tmp2, align 1
  br label %for.inc

for.inc:                                          ; preds = %if.then, %for.body
  %tmp6 = add nuw nsw i64 %tmp0, 1
  %tmp7 = add i64 %tmp1, -1
  %tmp8 = icmp eq i64 %tmp7, 0
  br i1 %tmp8, label %for.end, label %for.body

for.end:                                          ; preds = %for.inc
  ret void
}

opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S

opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S

opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S

opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S

opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=4 -force-widen-divrem-via-safe-divisor=0 -enable-interleaved-mem-accesses -S

opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -debug -disable-output %s 2>&1

opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s

define void @scalarize_ptrtoint(ptr %src, ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr ptr, ptr %src, i64 %iv
  %l = load ptr, ptr %gep, align 8
  %cast = ptrtoint ptr %l to i64
  %add = add i64 %cast, 10
  %cast.2 = inttoptr i64 %add to ptr
  store ptr %cast.2, ptr %dst, align 8
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

opt < %s -passes=loop-vectorize -force-vector-width=2 -S

define void @d() {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %i = phi i64 [ 0, %entry ], [ %i7, %loop ]
  %i3 = load float, ptr null, align 4
  %i4 = getelementptr float, ptr @d, i64 %i
  %i5 = tail call i1 @llvm.is.fpclass.f32(float %i3, i32 0)
  %i6 = select i1 %i5, float 0.000000e+00, float 0.000000e+00
  store float %i6, ptr %i4, align 4
  %i7 = add i64 %i, 1
  %i8 = icmp eq i64 %i7, 128
  br i1 %i8, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
declare i1 @llvm.is.fpclass.f32(float, i32 immarg) #0

attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }

opt -passes='loop-vectorize' -force-vector-width=4 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 -S %s

opt -passes='loop-vectorize' -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -S %s

target datalayout = "e-m:e-i64:64-n32:64-v128:128:128"

define void @multiple_ivs_wide(ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.2 = phi i32 [ -64, %entry ], [ %iv.2.next, %loop ]
  %iv.next = add i32 %iv, 2
  %iv.2.next = add i32 %iv.2, 2
  %gep = getelementptr inbounds i32, ptr %dst, i32 %iv
  store i32 %iv.2.next, ptr %gep, align 4
  %ec = icmp eq i32 %iv.next, 128
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

opt < %s -passes='loop-vectorize' -force-vector-interleave=1 -S -force-vector-width=4 -debug-only=loop-accesses,loop-vectorize,loop-utils

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"

define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) {
entry:
  %pre.c = icmp ugt i32 %Acols, 0
  br i1 %pre.c, label %exit, label %outer.header

outer.header:                                     ; preds = %outer.latch, %entry
  %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %mul.us = mul i32 %outer.iv, %Acols
  %arrayidx.us = getelementptr double, ptr %A, i32 %mul.us
  br label %inner

inner:                                            ; preds = %inner, %outer.header
  %inner.iv = phi i32 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %gep.C = getelementptr inbounds double, ptr %C, i32 %inner.iv
  %l = load double, ptr %arrayidx.us, align 8
  store double %l, ptr %gep.C, align 8
  %inner.iv.next = add i32 %inner.iv, 1
  %inner.c = icmp eq i32 %inner.iv.next, 1000
  br i1 %inner.c, label %outer.latch, label %inner

outer.latch:                                      ; preds = %inner
  %outer.iv.next = add i32 %outer.iv, 1
  %outer.c = icmp ult i32 %outer.iv, 128
  br i1 %outer.c, label %exit, label %outer.header

exit:                                             ; preds = %outer.latch, %entry
  ret void
}

opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -lv-strided-pointer-ivs=true -S %s

define void @step_direction_unknown(i32 %arg, ptr %dst) {
entry:
  %add = add i32 %arg, 1
  br label %loop

loop:                                             ; preds = %loop, %entry
  %phi = phi i64 [ 0, %entry ], [ %add2, %loop ]
  %trunc = trunc i64 %phi to i32
  %mul = mul i32 %add, %trunc
  %zext = zext i32 %mul to i64
  %getelementptr = getelementptr double, ptr %dst, i64 %zext
  store double 0.000000e+00, ptr %getelementptr, align 8
  %add2 = add i64 %phi, 1
  %icmp = icmp eq i64 %add2, 1024
  br i1 %icmp, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

define void @integer_induction_wraps_scev_predicate_known(i32 %x, ptr %call, ptr %start) {
entry:
  %mul = shl i32 %x, 1
  br label %for.cond

for.cond:                                         ; preds = %for.cond, %entry
  %iv = phi i32 [ 30, %entry ], [ %inc, %for.cond ]
  %p.0 = phi ptr [ %start, %entry ], [ %add.ptr, %for.cond ]
  %add.ptr = getelementptr i8, ptr %p.0, i32 %mul
  %arrayidx = getelementptr ptr, ptr %call, i32 %iv
  store ptr %p.0, ptr %arrayidx, align 4
  %inc = add i32 %iv, 1
  %tobool.not = icmp eq i32 %iv, 1024
  br i1 %tobool.not, label %for.end, label %for.cond

for.end:                                          ; preds = %for.cond
  ret void
}

opt -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 -debug -disable-output %s 2>&1

target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"

define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.inc, %entry
  %tmp0 = phi i64 [ %tmp6, %for.inc ], [ 0, %entry ]
  %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 1024, %entry ]
  %tmp2 = getelementptr i8, ptr %ptr, i64 %tmp0
  %tmp3 = load i8, ptr %tmp2, align 1
  store i8 0, ptr %tmp2, align 1
  %tmp4 = zext i8 %tmp3 to i32
  %tmp5 = trunc i32 %tmp4 to i8
  br i1 %c, label %if.then, label %for.inc

if.then:                                          ; preds = %for.body
  store i8 %tmp5, ptr %tmp2, align 1
  br label %for.inc

for.inc:                                          ; preds = %if.then, %for.body
  %tmp6 = add nuw nsw i64 %tmp0, 1
  %tmp7 = add i64 %tmp1, -1
  %tmp8 = icmp eq i64 %tmp7, 0
  br i1 %tmp8, label %for.end, label %for.body

for.end:                                          ; preds = %for.inc
  ret void
}

[LoopVectorizer]  Assertion `EPResumeVal && "must have a resume value for the canonical IV"' failed.

clang -Wno-incompatible-pointer-types -O3 -march=z13 -S -c crash19.i -o a.out -w -mllvm -disable-licm-promotion -mllvm -epilogue-vectorization-force-VF=2

[crash19.tar.gz](https://github.com/user-attachments/files/18383534/crash19.tar.gz)

#9 0x000002aa3ff5e440 preparePlanForEpilogueVectorLoop
#12 0x000002aa3ffa098a llvm::LoopVectorizePass::run

@bmahjour
@fhahn

@JonPsson1 Was the test case fuzzer generated? Just curious, because it had a loop with unsigned max as iteration count.

> [@JonPsson1](https://github.com/JonPsson1) Was the test case fuzzer generated? Just curious, because it had a loop with unsigned max as iteration count.

No, it was a CSmith test case.

@JonPsson1 Could you check if it fixes the full reproducer? If not, it would be great if you could share the full reproducer?