commit ea83e1c05a5adee5d8f9e680356ee57556ba64a1
Author: Florian Hahn <flo@fhahn.com>
Date:   Wed Sep 11 21:04:33 2024 +0100

    [LV] Assign cost to all interleave members when not interleaving.

    At the moment, the full cost of all interleave group members is assigned
    to the instruction at the group's insert position, even if the decision
    was to not form an interleave group.

    This can lead to inaccurate cost estimates, e.g. if the instruction at
    the insert position is dead. If the decision is to not vectorize but
    scalarize or scather/gather, then the cost will be to total cost for all
    members. In those cases, assign individual the cost per member, to more
    closely reflect to choice per instruction.

    This fixes a divergence between legacy and VPlan-based cost model.

    Fixes https://github.com/llvm/llvm-project/issues/108098.

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 640a7bf3d672..3b6b154b9660 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1182,13 +1182,23 @@ public:
                            InstructionCost Cost) {
     assert(VF.isVector() && "Expected VF >=2");
     /// Broadcast this decicion to all instructions inside the group.
-    /// But the cost will be assigned to one instruction only.
+    /// When interleaving, the cost will only be assigned one instruction, the
+    /// insert position. For other cases, add the appropriate fraction of the
+    /// total cost to each instruction. This ensures accurate costs are used,
+    /// even if the insert position instruction is not used.
+    InstructionCost InsertPosCost = Cost;
+    InstructionCost OtherMemberCost = 0;
+    if (W != CM_Interleave)
+      OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
+    ;
     for (unsigned Idx = 0; Idx < Grp->getFactor(); ++Idx) {
       if (auto *I = Grp->getMember(Idx)) {
         if (Grp->getInsertPos() == I)
-          WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
+          WideningDecisions[std::make_pair(I, VF)] =
+              std::make_pair(W, InsertPosCost);
         else
-          WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
+          WideningDecisions[std::make_pair(I, VF)] =
+              std::make_pair(W, OtherMemberCost);
       }
     }
   }

opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1

opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1

opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1

opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x float], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %in0 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.4
  %v0 = load float, ptr %in0, align 4
  %v1 = load float, ptr %in1, align 4
  %v2 = load float, ptr %in2, align 4
  %v3 = load float, ptr %in3, align 4
  %v4 = load float, ptr %in4, align 4
  %reduce.add.0 = fadd float %v0, %v1
  %reduce.add.1 = fadd float %reduce.add.0, %v2
  %reduce.add.2 = fadd float %reduce.add.1, %v3
  %reduce.add.3 = fadd float %reduce.add.2, %v4
  %reduce.add.3.narrow = fptoui float %reduce.add.3 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.3.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 5
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x float], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %in0 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.6
  %v0 = load float, ptr %in0, align 4
  %v1 = load float, ptr %in1, align 4
  %v2 = load float, ptr %in2, align 4
  %v3 = load float, ptr %in3, align 4
  %v4 = load float, ptr %in4, align 4
  %v5 = load float, ptr %in5, align 4
  %v6 = load float, ptr %in6, align 4
  %reduce.add.0 = fadd float %v0, %v1
  %reduce.add.1 = fadd float %reduce.add.0, %v2
  %reduce.add.2 = fadd float %reduce.add.1, %v3
  %reduce.add.3 = fadd float %reduce.add.2, %v4
  %reduce.add.4 = fadd float %reduce.add.3, %v5
  %reduce.add.5 = fadd float %reduce.add.4, %v6
  %reduce.add.5.narrow = fptoui float %reduce.add.5 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.5.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 7
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x float], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in0 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.6
  %in7 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %iv.7
  %v0 = load float, ptr %in0, align 4
  %v1 = load float, ptr %in1, align 4
  %v2 = load float, ptr %in2, align 4
  %v3 = load float, ptr %in3, align 4
  %v4 = load float, ptr %in4, align 4
  %v5 = load float, ptr %in5, align 4
  %v6 = load float, ptr %in6, align 4
  %v7 = load float, ptr %in7, align 4
  %reduce.add.0 = fadd float %v0, %v1
  %reduce.add.1 = fadd float %reduce.add.0, %v2
  %reduce.add.2 = fadd float %reduce.add.1, %v3
  %reduce.add.3 = fadd float %reduce.add.2, %v4
  %reduce.add.4 = fadd float %reduce.add.3, %v5
  %reduce.add.5 = fadd float %reduce.add.4, %v6
  %reduce.add.6 = fadd float %reduce.add.5, %v7
  %reduce.add.6.narrow = fptoui float %reduce.add.6 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.6.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.0.narrow = fptoui double %reduce.add.0 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.0.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 2
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.2
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %v2 = load double, ptr %in2, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.1 = fadd double %reduce.add.0, %v2
  %reduce.add.1.narrow = fptoui double %reduce.add.1 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.1.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 3
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.3
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %v2 = load double, ptr %in2, align 8
  %v3 = load double, ptr %in3, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.1 = fadd double %reduce.add.0, %v2
  %reduce.add.2 = fadd double %reduce.add.1, %v3
  %reduce.add.2.narrow = fptoui double %reduce.add.2 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.2.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 4
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.4
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %v2 = load double, ptr %in2, align 8
  %v3 = load double, ptr %in3, align 8
  %v4 = load double, ptr %in4, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.1 = fadd double %reduce.add.0, %v2
  %reduce.add.2 = fadd double %reduce.add.1, %v3
  %reduce.add.3 = fadd double %reduce.add.2, %v4
  %reduce.add.3.narrow = fptoui double %reduce.add.3 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.3.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 5
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.5
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %v2 = load double, ptr %in2, align 8
  %v3 = load double, ptr %in3, align 8
  %v4 = load double, ptr %in4, align 8
  %v5 = load double, ptr %in5, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.1 = fadd double %reduce.add.0, %v2
  %reduce.add.2 = fadd double %reduce.add.1, %v3
  %reduce.add.3 = fadd double %reduce.add.2, %v4
  %reduce.add.4 = fadd double %reduce.add.3, %v5
  %reduce.add.4.narrow = fptoui double %reduce.add.4 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.4.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 6
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.6
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %v2 = load double, ptr %in2, align 8
  %v3 = load double, ptr %in3, align 8
  %v4 = load double, ptr %in4, align 8
  %v5 = load double, ptr %in5, align 8
  %v6 = load double, ptr %in6, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.1 = fadd double %reduce.add.0, %v2
  %reduce.add.2 = fadd double %reduce.add.1, %v3
  %reduce.add.3 = fadd double %reduce.add.2, %v4
  %reduce.add.4 = fadd double %reduce.add.3, %v5
  %reduce.add.5 = fadd double %reduce.add.4, %v6
  %reduce.add.5.narrow = fptoui double %reduce.add.5 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.5.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 7
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x double], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in0 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.6
  %in7 = getelementptr inbounds [1024 x double], ptr @A, i64 0, i64 %iv.7
  %v0 = load double, ptr %in0, align 8
  %v1 = load double, ptr %in1, align 8
  %v2 = load double, ptr %in2, align 8
  %v3 = load double, ptr %in3, align 8
  %v4 = load double, ptr %in4, align 8
  %v5 = load double, ptr %in5, align 8
  %v6 = load double, ptr %in6, align 8
  %v7 = load double, ptr %in7, align 8
  %reduce.add.0 = fadd double %v0, %v1
  %reduce.add.1 = fadd double %reduce.add.0, %v2
  %reduce.add.2 = fadd double %reduce.add.1, %v3
  %reduce.add.3 = fadd double %reduce.add.2, %v4
  %reduce.add.4 = fadd double %reduce.add.3, %v5
  %reduce.add.5 = fadd double %reduce.add.4, %v6
  %reduce.add.6 = fadd double %reduce.add.5, %v7
  %reduce.add.6.narrow = fptoui double %reduce.add.6 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.6.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i16], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %in0 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.4
  %v0 = load i16, ptr %in0, align 2
  %v1 = load i16, ptr %in1, align 2
  %v2 = load i16, ptr %in2, align 2
  %v3 = load i16, ptr %in3, align 2
  %v4 = load i16, ptr %in4, align 2
  %reduce.add.0 = add i16 %v0, %v1
  %reduce.add.1 = add i16 %reduce.add.0, %v2
  %reduce.add.2 = add i16 %reduce.add.1, %v3
  %reduce.add.3 = add i16 %reduce.add.2, %v4
  %reduce.add.3.narrow = trunc i16 %reduce.add.3 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.3.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 5
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i16], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %in0 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.6
  %v0 = load i16, ptr %in0, align 2
  %v1 = load i16, ptr %in1, align 2
  %v2 = load i16, ptr %in2, align 2
  %v3 = load i16, ptr %in3, align 2
  %v4 = load i16, ptr %in4, align 2
  %v5 = load i16, ptr %in5, align 2
  %v6 = load i16, ptr %in6, align 2
  %reduce.add.0 = add i16 %v0, %v1
  %reduce.add.1 = add i16 %reduce.add.0, %v2
  %reduce.add.2 = add i16 %reduce.add.1, %v3
  %reduce.add.3 = add i16 %reduce.add.2, %v4
  %reduce.add.4 = add i16 %reduce.add.3, %v5
  %reduce.add.5 = add i16 %reduce.add.4, %v6
  %reduce.add.5.narrow = trunc i16 %reduce.add.5 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.5.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 7
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i16], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in0 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.6
  %in7 = getelementptr inbounds [1024 x i16], ptr @A, i64 0, i64 %iv.7
  %v0 = load i16, ptr %in0, align 2
  %v1 = load i16, ptr %in1, align 2
  %v2 = load i16, ptr %in2, align 2
  %v3 = load i16, ptr %in3, align 2
  %v4 = load i16, ptr %in4, align 2
  %v5 = load i16, ptr %in5, align 2
  %v6 = load i16, ptr %in6, align 2
  %v7 = load i16, ptr %in7, align 2
  %reduce.add.0 = add i16 %v0, %v1
  %reduce.add.1 = add i16 %reduce.add.0, %v2
  %reduce.add.2 = add i16 %reduce.add.1, %v3
  %reduce.add.3 = add i16 %reduce.add.2, %v4
  %reduce.add.4 = add i16 %reduce.add.3, %v5
  %reduce.add.5 = add i16 %reduce.add.4, %v6
  %reduce.add.6 = add i16 %reduce.add.5, %v7
  %reduce.add.6.narrow = trunc i16 %reduce.add.6 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.6.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i32], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %in0 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.1
  %v0 = load i32, ptr %in0, align 4
  %v1 = load i32, ptr %in1, align 4
  %reduce.add.0 = add i32 %v0, %v1
  %reduce.add.1 = add i32 %reduce.add.0, 0
  %reduce.add.2 = add i32 %reduce.add.1, 0
  %reduce.add.2.narrow = trunc i32 %reduce.add.2 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.2.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv, 4
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i32], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %in0 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.4
  %v0 = load i32, ptr %in0, align 4
  %v1 = load i32, ptr %in1, align 4
  %v2 = load i32, ptr %in2, align 4
  %v3 = load i32, ptr %in3, align 4
  %v4 = load i32, ptr %in4, align 4
  %reduce.add.0 = add i32 %v0, %v1
  %reduce.add.1 = add i32 %reduce.add.0, %v2
  %reduce.add.2 = add i32 %reduce.add.1, %v3
  %reduce.add.3 = add i32 %reduce.add.2, %v4
  %reduce.add.3.narrow = trunc i32 %reduce.add.3 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.3.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 5
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i32], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %in0 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.6
  %v0 = load i32, ptr %in0, align 4
  %v1 = load i32, ptr %in1, align 4
  %v2 = load i32, ptr %in2, align 4
  %v3 = load i32, ptr %in3, align 4
  %v4 = load i32, ptr %in4, align 4
  %v5 = load i32, ptr %in5, align 4
  %v6 = load i32, ptr %in6, align 4
  %reduce.add.0 = add i32 %v0, %v1
  %reduce.add.1 = add i32 %reduce.add.0, %v2
  %reduce.add.2 = add i32 %reduce.add.1, %v3
  %reduce.add.3 = add i32 %reduce.add.2, %v4
  %reduce.add.4 = add i32 %reduce.add.3, %v5
  %reduce.add.5 = add i32 %reduce.add.4, %v6
  %reduce.add.5.narrow = trunc i32 %reduce.add.5 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.5.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 7
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i32], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in0 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.6
  %in7 = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %iv.7
  %v0 = load i32, ptr %in0, align 4
  %v1 = load i32, ptr %in1, align 4
  %v2 = load i32, ptr %in2, align 4
  %v3 = load i32, ptr %in3, align 4
  %v4 = load i32, ptr %in4, align 4
  %v5 = load i32, ptr %in5, align 4
  %v6 = load i32, ptr %in6, align 4
  %v7 = load i32, ptr %in7, align 4
  %reduce.add.0 = add i32 %v0, %v1
  %reduce.add.1 = add i32 %reduce.add.0, %v2
  %reduce.add.2 = add i32 %reduce.add.1, %v3
  %reduce.add.3 = add i32 %reduce.add.2, %v4
  %reduce.add.4 = add i32 %reduce.add.3, %v5
  %reduce.add.5 = add i32 %reduce.add.4, %v6
  %reduce.add.6 = add i32 %reduce.add.5, %v7
  %reduce.add.6.narrow = trunc i32 %reduce.add.6 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.6.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.0.narrow = trunc i64 %reduce.add.0 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.0.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 2
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.2
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %v2 = load i64, ptr %in2, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.1 = add i64 %reduce.add.0, %v2
  %reduce.add.1.narrow = trunc i64 %reduce.add.1 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.1.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 3
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.3
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %v2 = load i64, ptr %in2, align 8
  %v3 = load i64, ptr %in3, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.1 = add i64 %reduce.add.0, %v2
  %reduce.add.2 = add i64 %reduce.add.1, %v3
  %reduce.add.2.narrow = trunc i64 %reduce.add.2 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.2.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 4
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.4
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %v2 = load i64, ptr %in2, align 8
  %v3 = load i64, ptr %in3, align 8
  %v4 = load i64, ptr %in4, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.1 = add i64 %reduce.add.0, %v2
  %reduce.add.2 = add i64 %reduce.add.1, %v3
  %reduce.add.3 = add i64 %reduce.add.2, %v4
  %reduce.add.3.narrow = trunc i64 %reduce.add.3 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.3.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 5
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.5
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %v2 = load i64, ptr %in2, align 8
  %v3 = load i64, ptr %in3, align 8
  %v4 = load i64, ptr %in4, align 8
  %v5 = load i64, ptr %in5, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.1 = add i64 %reduce.add.0, %v2
  %reduce.add.2 = add i64 %reduce.add.1, %v3
  %reduce.add.3 = add i64 %reduce.add.2, %v4
  %reduce.add.4 = add i64 %reduce.add.3, %v5
  %reduce.add.4.narrow = trunc i64 %reduce.add.4 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.4.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 6
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.6
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %v2 = load i64, ptr %in2, align 8
  %v3 = load i64, ptr %in3, align 8
  %v4 = load i64, ptr %in4, align 8
  %v5 = load i64, ptr %in5, align 8
  %v6 = load i64, ptr %in6, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.1 = add i64 %reduce.add.0, %v2
  %reduce.add.2 = add i64 %reduce.add.1, %v3
  %reduce.add.3 = add i64 %reduce.add.2, %v4
  %reduce.add.4 = add i64 %reduce.add.3, %v5
  %reduce.add.5 = add i64 %reduce.add.4, %v6
  %reduce.add.5.narrow = trunc i64 %reduce.add.5 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.5.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 7
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i64], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in0 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.6
  %in7 = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.7
  %v0 = load i64, ptr %in0, align 8
  %v1 = load i64, ptr %in1, align 8
  %v2 = load i64, ptr %in2, align 8
  %v3 = load i64, ptr %in3, align 8
  %v4 = load i64, ptr %in4, align 8
  %v5 = load i64, ptr %in5, align 8
  %v6 = load i64, ptr %in6, align 8
  %v7 = load i64, ptr %in7, align 8
  %reduce.add.0 = add i64 %v0, %v1
  %reduce.add.1 = add i64 %reduce.add.0, %v2
  %reduce.add.2 = add i64 %reduce.add.1, %v3
  %reduce.add.3 = add i64 %reduce.add.2, %v4
  %reduce.add.4 = add i64 %reduce.add.3, %v5
  %reduce.add.5 = add i64 %reduce.add.4, %v6
  %reduce.add.6 = add i64 %reduce.add.5, %v7
  %reduce.add.6.narrow = trunc i64 %reduce.add.6 to i8
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.6.narrow, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i8], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %in0 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.4
  %v0 = load i8, ptr %in0, align 1
  %v1 = load i8, ptr %in1, align 1
  %v2 = load i8, ptr %in2, align 1
  %v3 = load i8, ptr %in3, align 1
  %v4 = load i8, ptr %in4, align 1
  %reduce.add.0 = add i8 %v0, %v1
  %reduce.add.1 = add i8 %reduce.add.0, %v2
  %reduce.add.2 = add i8 %reduce.add.1, %v3
  %reduce.add.3 = add i8 %reduce.add.2, %v4
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.3, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 5
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i8], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %in0 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.6
  %v0 = load i8, ptr %in0, align 1
  %v1 = load i8, ptr %in1, align 1
  %v2 = load i8, ptr %in2, align 1
  %v3 = load i8, ptr %in3, align 1
  %v4 = load i8, ptr %in4, align 1
  %v5 = load i8, ptr %in5, align 1
  %v6 = load i8, ptr %in6, align 1
  %reduce.add.0 = add i8 %v0, %v1
  %reduce.add.1 = add i8 %reduce.add.0, %v2
  %reduce.add.2 = add i8 %reduce.add.1, %v3
  %reduce.add.3 = add i8 %reduce.add.2, %v4
  %reduce.add.4 = add i8 %reduce.add.3, %v5
  %reduce.add.5 = add i8 %reduce.add.4, %v6
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.5, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 7
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i8], align 128
@B = external global [1024 x i8], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in0 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.0
  %in1 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.1
  %in2 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.2
  %in3 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.3
  %in4 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.4
  %in5 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.5
  %in6 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.6
  %in7 = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.7
  %v0 = load i8, ptr %in0, align 1
  %v1 = load i8, ptr %in1, align 1
  %v2 = load i8, ptr %in2, align 1
  %v3 = load i8, ptr %in3, align 1
  %v4 = load i8, ptr %in4, align 1
  %v5 = load i8, ptr %in5, align 1
  %v6 = load i8, ptr %in6, align 1
  %v7 = load i8, ptr %in7, align 1
  %reduce.add.0 = add i8 %v0, %v1
  %reduce.add.1 = add i8 %reduce.add.0, %v2
  %reduce.add.2 = add i8 %reduce.add.1, %v3
  %reduce.add.3 = add i8 %reduce.add.2, %v4
  %reduce.add.4 = add i8 %reduce.add.3, %v5
  %reduce.add.5 = add i8 %reduce.add.4, %v6
  %reduce.add.6 = add i8 %reduce.add.5, %v7
  %out = getelementptr inbounds [1024 x i8], ptr @B, i64 0, i64 %iv.0
  store i8 %reduce.add.6, ptr %out, align 1
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i8], align 128
@B = external global [1024 x double], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.0
  %v.narrow = load i8, ptr %in, align 1
  %v = uitofp i8 %v.narrow to double
  %v0 = fadd double %v, 0.000000e+00
  %v1 = fadd double %v, 1.000000e+00
  %v2 = fadd double %v, 2.000000e+00
  %v3 = fadd double %v, 3.000000e+00
  %v4 = fadd double %v, 4.000000e+00
  %v5 = fadd double %v, 5.000000e+00
  %v6 = fadd double %v, 6.000000e+00
  %v7 = fadd double %v, 7.000000e+00
  %out0 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.0
  %out1 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.1
  %out2 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.2
  %out3 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.3
  %out4 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.4
  %out5 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.5
  %out6 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.6
  %out7 = getelementptr inbounds [1024 x double], ptr @B, i64 0, i64 %iv.7
  store double %v0, ptr %out0, align 8
  store double %v1, ptr %out1, align 8
  store double %v2, ptr %out2, align 8
  store double %v3, ptr %out3, align 8
  store double %v4, ptr %out4, align 8
  store double %v5, ptr %out5, align 8
  store double %v6, ptr %out6, align 8
  store double %v7, ptr %out7, align 8
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@A = external global [1024 x i8], align 128
@B = external global [1024 x i64], align 128

define void @test() {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv.0 = add nuw nsw i64 %iv, 0
  %iv.1 = add nuw nsw i64 %iv, 1
  %iv.2 = add nuw nsw i64 %iv, 2
  %iv.3 = add nuw nsw i64 %iv, 3
  %iv.4 = add nuw nsw i64 %iv, 4
  %iv.5 = add nuw nsw i64 %iv, 5
  %iv.6 = add nuw nsw i64 %iv, 6
  %iv.7 = add nuw nsw i64 %iv, 7
  %in = getelementptr inbounds [1024 x i8], ptr @A, i64 0, i64 %iv.0
  %v.narrow = load i8, ptr %in, align 1
  %v = zext i8 %v.narrow to i64
  %v0 = add i64 %v, 0
  %v1 = add i64 %v, 1
  %v2 = add i64 %v, 2
  %v3 = add i64 %v, 3
  %v4 = add i64 %v, 4
  %v5 = add i64 %v, 5
  %v6 = add i64 %v, 6
  %v7 = add i64 %v, 7
  %out0 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.0
  %out1 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.1
  %out2 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.2
  %out3 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.3
  %out4 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.4
  %out5 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.5
  %out6 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.6
  %out7 = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.7
  store i64 %v0, ptr %out0, align 8
  store i64 %v1, ptr %out1, align 8
  store i64 %v2, ptr %out2, align 8
  store i64 %v3, ptr %out3, align 8
  store i64 %v4, ptr %out4, align 8
  store i64 %v5, ptr %out5, align 8
  store i64 %v6, ptr %out6, align 8
  store i64 %v7, ptr %out7, align 8
  %iv.next = add nuw nsw i64 %iv.0, 8
  %cmp = icmp ult i64 %iv.next, 1024
  br i1 %cmp, label %for.body, label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %for.body
  ret void
}

opt -passes=loop-vectorize -enable-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize < %s 2>&1

opt -passes=loop-vectorize -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize < %s 2>&1

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y) {
entry:
  %cmp15 = icmp sgt i32 %numPoints, 0
  br i1 %cmp15, label %for.body.preheader, label %for.end

for.body.preheader:                               ; preds = %entry
  %wide.trip.count = zext i32 %numPoints to i64
  br label %for.body

for.body:                                         ; preds = %for.body, %for.body.preheader
  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
  %arrayidx = getelementptr inbounds i16, ptr %x, i64 %indvars.iv
  %0 = load i16, ptr %arrayidx, align 2
  %1 = shl nsw i64 %indvars.iv, 2
  %arrayidx2 = getelementptr inbounds i16, ptr %points, i64 %1
  store i16 %0, ptr %arrayidx2, align 2
  %arrayidx4 = getelementptr inbounds i16, ptr %y, i64 %indvars.iv
  %2 = load i16, ptr %arrayidx4, align 2
  %3 = or disjoint i64 %1, 1
  %arrayidx7 = getelementptr inbounds i16, ptr %points, i64 %3
  store i16 %2, ptr %arrayidx7, align 2
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
  br i1 %exitcond.not, label %for.end.loopexit, label %for.body

for.end.loopexit:                                 ; preds = %for.body
  br label %for.end

for.end:                                          ; preds = %for.end.loopexit, %entry
  ret void
}

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define void @test1(ptr noalias nocapture %points, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
  %arrayidx = getelementptr inbounds i16, ptr %x, i64 %indvars.iv
  %0 = load i16, ptr %arrayidx, align 2
  %1 = shl nuw nsw i64 %indvars.iv, 2
  %arrayidx2 = getelementptr inbounds i16, ptr %points, i64 %1
  store i16 %0, ptr %arrayidx2, align 2
  %arrayidx4 = getelementptr inbounds i16, ptr %y, i64 %indvars.iv
  %2 = load i16, ptr %arrayidx4, align 2
  %3 = or disjoint i64 %1, 1
  %arrayidx7 = getelementptr inbounds i16, ptr %points, i64 %3
  store i16 %2, ptr %arrayidx7, align 2
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:                                          ; preds = %for.body
  ret void
}

opt < %s -force-vector-width=2 -force-vector-interleave=1 -passes=loop-vectorize -S --debug-only=loop-vectorize 2>&1

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"

%pair = type { i8, i8 }

define void @test(ptr %p, ptr %q, i64 %n) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr %pair, ptr %p, i64 %i, i32 0
  %tmp1 = load i8, ptr %tmp0, align 1
  %tmp2 = getelementptr %pair, ptr %p, i64 %i, i32 1
  %tmp3 = load i8, ptr %tmp2, align 1
  %add = add i8 %tmp1, %tmp3
  %qi = getelementptr i8, ptr %q, i64 %i
  store i8 %add, ptr %qi, align 1
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp eq i64 %i.next, %n
  br i1 %cond, label %for.end, label %for.body

for.end:                                          ; preds = %for.body
  ret void
}

opt -passes=loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1

opt -passes=loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1

opt -passes=loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1

opt -passes=loop-vectorize -force-vector-width=16 -debug-only=loop-vectorize -disable-output < %s 2>&1

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"

%i8.2 = type { i8, i8 }

define void @i8_factor_2(ptr %data, i64 %n) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i8, ptr %tmp0, align 1
  %tmp3 = load i8, ptr %tmp1, align 1
  store i8 %tmp2, ptr %tmp0, align 1
  store i8 %tmp3, ptr %tmp1, align 1
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"

%i16.2 = type { i16, i16 }

define void @i16_factor_2(ptr %data, i64 %n) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i16, ptr %tmp0, align 2
  %tmp3 = load i16, ptr %tmp1, align 2
  store i16 %tmp2, ptr %tmp0, align 2
  store i16 %tmp3, ptr %tmp1, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"

%i32.2 = type { i32, i32 }

define void @i32_factor_2(ptr %data, i64 %n) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i32, ptr %tmp0, align 4
  %tmp3 = load i32, ptr %tmp1, align 4
  store i32 %tmp2, ptr %tmp0, align 4
  store i32 %tmp3, ptr %tmp1, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"

%i64.2 = type { i64, i64 }

define void @i64_factor_2(ptr %data, i64 %n) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i64, ptr %tmp0, align 8
  %tmp3 = load i64, ptr %tmp1, align 8
  store i64 %tmp2, ptr %tmp0, align 8
  store i64 %tmp3, ptr %tmp1, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"

%i64.8 = type { i64, i64, i64, i64, i64, i64, i64, i64 }

define void @i64_factor_8(ptr %data, i64 %n) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i64.8, ptr %data, i64 %i, i32 2
  %tmp1 = getelementptr inbounds %i64.8, ptr %data, i64 %i, i32 6
  %tmp2 = load i64, ptr %tmp0, align 8
  %tmp3 = load i64, ptr %tmp1, align 8
  store i64 %tmp2, ptr %tmp0, align 8
  store i64 %tmp3, ptr %tmp1, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i16.3 = type { i16, i16, i16 }

define void @i16_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 2
  %tmp3 = load i16, ptr %tmp0, align 2
  %tmp4 = load i16, ptr %tmp1, align 2
  %tmp5 = load i16, ptr %tmp2, align 2
  store i16 %tmp3, ptr %tmp0, align 2
  store i16 %tmp4, ptr %tmp1, align 2
  store i16 %tmp5, ptr %tmp2, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f16.2 = type { half, half }

define void @f16_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f16.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f16.2, ptr %data, i64 %i, i32 1
  %tmp2 = load half, ptr %tmp0, align 2
  %tmp3 = load half, ptr %tmp1, align 2
  store half %tmp2, ptr %tmp0, align 2
  store half %tmp3, ptr %tmp1, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i32.2 = type { i32, i32 }

define void @i32_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i32, ptr %tmp0, align 4
  %tmp3 = load i32, ptr %tmp1, align 4
  store i32 %tmp2, ptr %tmp0, align 4
  store i32 %tmp3, ptr %tmp1, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i64.4 = type { i64, i64, i64, i64 }

define void @i64_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i64.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i64.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i64.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %i64.4, ptr %data, i64 %i, i32 3
  %tmp4 = load i64, ptr %tmp0, align 8
  %tmp5 = load i64, ptr %tmp1, align 8
  %tmp6 = load i64, ptr %tmp2, align 8
  %tmp7 = load i64, ptr %tmp3, align 8
  store i64 %tmp4, ptr %tmp0, align 8
  store i64 %tmp5, ptr %tmp1, align 8
  store i64 %tmp6, ptr %tmp2, align 8
  store i64 %tmp7, ptr %tmp3, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f32.3 = type { float, float, float }

define void @f32_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f32.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f32.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %f32.3, ptr %data, i64 %i, i32 2
  %tmp3 = load float, ptr %tmp0, align 4
  %tmp4 = load float, ptr %tmp1, align 4
  %tmp5 = load float, ptr %tmp2, align 4
  store float %tmp3, ptr %tmp0, align 4
  store float %tmp4, ptr %tmp1, align 4
  store float %tmp5, ptr %tmp2, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i64.3 = type { i64, i64, i64 }

define void @i64_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 2
  %tmp3 = load i64, ptr %tmp0, align 8
  %tmp4 = load i64, ptr %tmp1, align 8
  %tmp5 = load i64, ptr %tmp2, align 8
  store i64 %tmp3, ptr %tmp0, align 8
  store i64 %tmp4, ptr %tmp1, align 8
  store i64 %tmp5, ptr %tmp2, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f16.3 = type { half, half, half }

define void @f16_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f16.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f16.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %f16.3, ptr %data, i64 %i, i32 2
  %tmp3 = load half, ptr %tmp0, align 2
  %tmp4 = load half, ptr %tmp1, align 2
  %tmp5 = load half, ptr %tmp2, align 2
  store half %tmp3, ptr %tmp0, align 2
  store half %tmp4, ptr %tmp1, align 2
  store half %tmp5, ptr %tmp2, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i8.3 = type { i8, i8, i8 }

define void @i8_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 2
  %tmp3 = load i8, ptr %tmp0, align 1
  %tmp4 = load i8, ptr %tmp1, align 1
  %tmp5 = load i8, ptr %tmp2, align 1
  store i8 %tmp3, ptr %tmp0, align 1
  store i8 %tmp4, ptr %tmp1, align 1
  store i8 %tmp5, ptr %tmp2, align 1
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f32.2 = type { float, float }

define void @f32_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f32.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f32.2, ptr %data, i64 %i, i32 1
  %tmp2 = load float, ptr %tmp0, align 4
  %tmp3 = load float, ptr %tmp1, align 4
  store float %tmp2, ptr %tmp0, align 4
  store float %tmp3, ptr %tmp1, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i8.4 = type { i8, i8, i8, i8 }

define void @i8_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 3
  %tmp4 = load i8, ptr %tmp0, align 1
  %tmp5 = load i8, ptr %tmp1, align 1
  %tmp6 = load i8, ptr %tmp2, align 1
  %tmp7 = load i8, ptr %tmp3, align 1
  store i8 %tmp4, ptr %tmp0, align 1
  store i8 %tmp5, ptr %tmp1, align 1
  store i8 %tmp6, ptr %tmp2, align 1
  store i8 %tmp7, ptr %tmp3, align 1
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i16.2 = type { i16, i16 }

define void @i16_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i16, ptr %tmp0, align 2
  %tmp3 = load i16, ptr %tmp1, align 2
  store i16 %tmp2, ptr %tmp0, align 2
  store i16 %tmp3, ptr %tmp1, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i16.4 = type { i16, i16, i16, i16 }

define void @i16_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i16.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i16.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i16.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %i16.4, ptr %data, i64 %i, i32 3
  %tmp4 = load i16, ptr %tmp0, align 2
  %tmp5 = load i16, ptr %tmp1, align 2
  %tmp6 = load i16, ptr %tmp2, align 2
  %tmp7 = load i16, ptr %tmp3, align 2
  store i16 %tmp4, ptr %tmp0, align 2
  store i16 %tmp5, ptr %tmp1, align 2
  store i16 %tmp6, ptr %tmp2, align 2
  store i16 %tmp7, ptr %tmp3, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f64.3 = type { double, double, double }

define void @f64_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f64.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f64.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %f64.3, ptr %data, i64 %i, i32 2
  %tmp3 = load double, ptr %tmp0, align 8
  %tmp4 = load double, ptr %tmp1, align 8
  %tmp5 = load double, ptr %tmp2, align 8
  store double %tmp3, ptr %tmp0, align 8
  store double %tmp4, ptr %tmp1, align 8
  store double %tmp5, ptr %tmp2, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i8.2 = type { i8, i8 }

define void @i8_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i8, ptr %tmp0, align 1
  %tmp3 = load i8, ptr %tmp1, align 1
  store i8 %tmp2, ptr %tmp0, align 1
  store i8 %tmp3, ptr %tmp1, align 1
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i32.4 = type { i32, i32, i32, i32 }

define void @i32_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i32.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i32.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i32.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %i32.4, ptr %data, i64 %i, i32 3
  %tmp4 = load i32, ptr %tmp0, align 4
  %tmp5 = load i32, ptr %tmp1, align 4
  %tmp6 = load i32, ptr %tmp2, align 4
  %tmp7 = load i32, ptr %tmp3, align 4
  store i32 %tmp4, ptr %tmp0, align 4
  store i32 %tmp5, ptr %tmp1, align 4
  store i32 %tmp6, ptr %tmp2, align 4
  store i32 %tmp7, ptr %tmp3, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i32.3 = type { i32, i32, i32 }

define void @i32_factor_3(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 2
  %tmp3 = load i32, ptr %tmp0, align 4
  %tmp4 = load i32, ptr %tmp1, align 4
  %tmp5 = load i32, ptr %tmp2, align 4
  store i32 %tmp3, ptr %tmp0, align 4
  store i32 %tmp4, ptr %tmp1, align 4
  store i32 %tmp5, ptr %tmp2, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f64.2 = type { double, double }

define void @f64_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f64.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f64.2, ptr %data, i64 %i, i32 1
  %tmp2 = load double, ptr %tmp0, align 8
  %tmp3 = load double, ptr %tmp1, align 8
  store double %tmp2, ptr %tmp0, align 8
  store double %tmp3, ptr %tmp1, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f16.4 = type { half, half, half, half }

define void @f16_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f16.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f16.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %f16.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %f16.4, ptr %data, i64 %i, i32 3
  %tmp4 = load half, ptr %tmp0, align 2
  %tmp5 = load half, ptr %tmp1, align 2
  %tmp6 = load half, ptr %tmp2, align 2
  %tmp7 = load half, ptr %tmp3, align 2
  store half %tmp4, ptr %tmp0, align 2
  store half %tmp5, ptr %tmp1, align 2
  store half %tmp6, ptr %tmp2, align 2
  store half %tmp7, ptr %tmp3, align 2
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%i64.2 = type { i64, i64 }

define void @i64_factor_2(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 1
  %tmp2 = load i64, ptr %tmp0, align 8
  %tmp3 = load i64, ptr %tmp1, align 8
  store i64 %tmp2, ptr %tmp0, align 8
  store i64 %tmp3, ptr %tmp1, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f64.4 = type { double, double, double, double }

define void @f64_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f64.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f64.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %f64.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %f64.4, ptr %data, i64 %i, i32 3
  %tmp4 = load double, ptr %tmp0, align 8
  %tmp5 = load double, ptr %tmp1, align 8
  %tmp6 = load double, ptr %tmp2, align 8
  %tmp7 = load double, ptr %tmp3, align 8
  store double %tmp4, ptr %tmp0, align 8
  store double %tmp5, ptr %tmp1, align 8
  store double %tmp6, ptr %tmp2, align 8
  store double %tmp7, ptr %tmp3, align 8
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-eabi"

%f32.4 = type { float, float, float, float }

define void @f32_factor_4(ptr %data, i64 %n) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
  %tmp0 = getelementptr inbounds %f32.4, ptr %data, i64 %i, i32 0
  %tmp1 = getelementptr inbounds %f32.4, ptr %data, i64 %i, i32 1
  %tmp2 = getelementptr inbounds %f32.4, ptr %data, i64 %i, i32 2
  %tmp3 = getelementptr inbounds %f32.4, ptr %data, i64 %i, i32 3
  %tmp4 = load float, ptr %tmp0, align 4
  %tmp5 = load float, ptr %tmp1, align 4
  %tmp6 = load float, ptr %tmp2, align 4
  %tmp7 = load float, ptr %tmp3, align 4
  store float %tmp4, ptr %tmp0, align 4
  store float %tmp5, ptr %tmp1, align 4
  store float %tmp6, ptr %tmp2, align 4
  store float %tmp7, ptr %tmp3, align 4
  %i.next = add nuw nsw i64 %i, 1
  %cond = icmp slt i64 %i.next, %n
  br i1 %cond, label %for.body, label %for.end

for.end:                                          ; preds = %for.body
  ret void
}

attributes #0 = { "target-features"="+mve.fp" }

opt -p loop-vectorize -mtriple riscv64-linux-gnu -mattr=+v,+f -S %s

target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"

define void @dead_load(ptr %p, i16 %start) {
entry:
  %start.ext = sext i16 %start to i64
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ %start.ext, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr i16, ptr %p, i64 %iv
  store i16 0, ptr %gep, align 2
  %l = load i16, ptr %gep, align 2
  %iv.next = add i64 %iv, 3
  %cmp = icmp slt i64 %iv, 111
  br i1 %cmp, label %loop, label %exit

exit:                                             ; preds = %loop
  ret void
}

target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"

; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
declare i16 @llvm.umax.i16(i16, i16) #0

define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #1 {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 %c, label %then, label %loop.exiting

then:                                             ; preds = %loop.header
  %gep = getelementptr inbounds i32, ptr %b, i32 %iv
  store i1 false, ptr %a, align 1
  store i32 0, ptr %gep, align 4
  br label %loop.exiting

loop.exiting:                                     ; preds = %then, %loop.header
  %iv.next = add i32 %iv, 1
  %umax = tail call i16 @llvm.umax.i16(i16 %x, i16 111)
  %umax.ext = zext i16 %umax to i32
  %sub = sub i32 770, %umax.ext
  %ec = icmp slt i32 %iv, %sub
  br i1 %ec, label %loop.latch, label %exit

loop.latch:                                       ; preds = %loop.exiting
  br label %loop.header

exit:                                             ; preds = %loop.exiting
  br label %return

return:                                           ; preds = %exit
  ret i32 0
}

attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
attributes #1 = { "target-features"="+64bit,+v" }

target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"

define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %src, ptr noalias %dst) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
  %gep.src.0 = getelementptr i8, ptr %src, i64 %iv
  %l.dead = load i8, ptr %gep.src.0, align 1
  %iv.1 = add i64 %iv, 1
  %gep.src.1 = getelementptr i8, ptr %src, i64 %iv.1
  %l.1 = load i8, ptr %gep.src.1, align 1
  %ext = zext i8 %l.1 to i32
  %gep.dst = getelementptr i32, ptr %dst, i64 %iv
  store i32 %ext, ptr %gep.dst, align 4
  %iv.next = add nsw i64 %iv, 4
  %ec = icmp slt i64 %iv, %N
  br i1 %ec, label %loop, label %exit

exit:                                             ; preds = %loop
  ret void
}

attributes #0 = { "target-features"="+64bit,+v" }

target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"

define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 false, label %loop.latch, label %then

then:                                             ; preds = %loop.header
  %not.a = xor i32 %a, -1
  br label %loop.latch

loop.latch:                                       ; preds = %then, %loop.header
  %p = phi i32 [ %not.a, %then ], [ 0, %loop.header ]
  %gep = getelementptr i32, ptr %dst, i64 %iv
  store i32 %p, ptr %gep, align 4
  %iv.next = add i64 %iv, 9
  %ec = icmp slt i64 %iv, 322
  br i1 %ec, label %loop.header, label %exit

exit:                                             ; preds = %loop.latch
  ret void
}

target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"

define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %idxprom = sext i32 %iv to i64
  %gep.src = getelementptr i8, ptr %src, i64 %idxprom
  %l = load i8, ptr %gep.src, align 1
  %gep.dst = getelementptr i8, ptr %dst, i64 %idxprom
  store i8 0, ptr %gep.dst, align 1
  %iv.next = add i32 %iv, 4
  %cmp = icmp ult i32 %iv, 1001
  br i1 %cmp, label %loop, label %exit

exit:                                             ; preds = %loop
  %r = phi i8 [ %l, %loop ]
  ret i8 %r
}

opt -mtriple=s390x-unknown-linux -mcpu=z16 -passes=loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 -disable-output < %s 2>&1

define noundef i32 @fun(i32 %argc, ptr nocapture readnone %argv) {
entry:
  %l_4774.i = alloca [4 x [2 x i128]], align 8
  call void @llvm.lifetime.start.p0(i64 128, ptr nonnull %l_4774.i)
  br label %for.cond4.preheader.i

for.cond4.preheader.i:                            ; preds = %for.cond4.preheader.i, %entry
  %indvars.iv8.i = phi i64 [ 0, %entry ], [ %indvars.iv.next9.i, %for.cond4.preheader.i ]
  %arrayidx10.i = getelementptr inbounds [4 x [2 x i128]], ptr %l_4774.i, i64 0, i64 %indvars.iv8.i, i64 0
  store i128 8721036757475490113, ptr %arrayidx10.i, align 8
  %arrayidx10.i.c = getelementptr inbounds [4 x [2 x i128]], ptr %l_4774.i, i64 0, i64 %indvars.iv8.i, i64 1
  store i128 8721036757475490113, ptr %arrayidx10.i.c, align 8
  %indvars.iv.next9.i = add nuw nsw i64 %indvars.iv8.i, 1
  %exitcond.not.i = icmp eq i64 %indvars.iv.next9.i, 4
  br i1 %exitcond.not.i, label %func_1.exit, label %for.cond4.preheader.i

func_1.exit:                                      ; preds = %for.cond4.preheader.i
  %arrayidx195.i = getelementptr inbounds [4 x [2 x i128]], ptr %l_4774.i, i64 0, i64 1
  %0 = load i128, ptr %arrayidx195.i, align 8
  %cmp200.i = icmp ne i128 %0, 0
  %conv202.i = zext i1 %cmp200.i to i64
  %call203.i = tail call i64 @safe_sub_func_int64_t_s_s(i64 noundef %conv202.i, i64 noundef 9139899272418802852)
  call void @llvm.lifetime.end.p0(i64 128, ptr nonnull %l_4774.i)
  br label %for.cond

for.cond:                                         ; preds = %for.cond, %func_1.exit
  br label %for.cond
}

; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0

; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #0

declare dso_local i64 @safe_sub_func_int64_t_s_s(i64, i64)

attributes #0 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }

[VPlan] Assertion " VPlan cost model and legacy cost model disagreed"' failed.

Tested using: 15106c26662a WITH commits from #107894 cherry-picked on top.
```
> git log --oneline
2bfba9bc7d37 (HEAD -> main) !fixup add TODO
28acd6af75ff Add initial loop-invariant code motion transform.
15106c26662a (origin/main, origin/HEAD) [flang][runtime] Fix odd "invalid descriptor" runtime crash
```
Testcase:
```c
char a;
extern int b[];
short c;
char d[1];
void checksum() {
  for (long e = 0; e < 3;)
    for (long f = 0; e < 3; e = 709692)
      for (; f < c; f += 4) {
        a = d[f];
        b[f] = d[f - 1];
      }
}
```
`clang -march=rv64gcv -O3 red.c`

Testcase:
```llvm ir
; ModuleID = 'reduced.bc'
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
target triple = "riscv64-unknown-linux-gnu"

define void @g(i64 %conv) #0 {
entry:
  br label %for.body8.us23

for.body8.us23:                                   ; preds = %for.body8.us23, %entry
  %f.121.us24 = phi i64 [ %add.us30, %for.body8.us23 ], [ 0, %entry ]
  %arrayidx.us25 = getelementptr [1 x i8], ptr null, i64 0, i64 %f.121.us24
  %0 = load i8, ptr %arrayidx.us25, align 1
  %sub.us26 = add i64 %f.121.us24, 1
  %arrayidx9.us27 = getelementptr [1 x i8], ptr null, i64 0, i64 %sub.us26
  %1 = load i8, ptr %arrayidx9.us27, align 1
  %conv10.us28 = zext i8 %1 to i32
  %arrayidx11.us29 = getelementptr [0 x i32], ptr null, i64 0, i64 %f.121.us24
  store i32 %conv10.us28, ptr null, align 4
  %add.us30 = add nsw i64 %f.121.us24, 4
  %cmp6.us31 = icmp slt i64 %f.121.us24, %conv
  br i1 %cmp6.us31, label %for.body8.us23, label %for.cond.cleanup.split.us

for.cond.cleanup.split.us:                        ; preds = %for.body8.us23
  ret void
}

attributes #0 = { "target-features"="+64bit,+a,+c,+d,+f,+m,+relax,+v,+zicsr,+zifencei,+zmmul,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-b,-e,-experimental-smctr,-experimental-smmpm,-experimental-smnpm,-experimental-ssctr,-experimental-ssnpm,-experimental-sspm,-experimental-supm,-experimental-zacas,-experimental-zalasr,-experimental-zicfilp,-experimental-zicfiss,-experimental-zvbc32e,-experimental-zvkgs,-h,-shcounterenw,-shgatpa,-shtvala,-shvsatpa,-shvstvala,-shvstvecd,-smaia,-smcdeleg,-smcsrind,-smepmp,-smstateen,-ssaia,-ssccfg,-ssccptr,-sscofpmf,-sscounterenw,-sscsrind,-ssqosid,-ssstateen,-ssstrict,-sstc,-sstvala,-sstvecd,-ssu64xl,-svade,-svadu,-svbare,-svinval,-svnapot,-svpbmt,-xcvalu,-xcvbi,-xcvbitmanip,-xcvelw,-xcvmac,-xcvmem,-xcvsimd,-xsfcease,-xsfvcp,-xsfvfnrclipxfqf,-xsfvfwmaccqqq,-xsfvqmaccdod,-xsfvqmaccqoq,-xsifivecdiscarddlone,-xsifivecflushdlone,-xtheadba,-xtheadbb,-xtheadbs,-xtheadcmo,-xtheadcondmov,-xtheadfmemidx,-xtheadmac,-xtheadmemidx,-xtheadmempair,-xtheadsync,-xtheadvdot,-xventanacondops,-xwchc,-za128rs,-za64rs,-zaamo,-zabha,-zalrsc,-zama16b,-zawrs,-zba,-zbb,-zbc,-zbkb,-zbkc,-zbkx,-zbs,-zca,-zcb,-zcd,-zce,-zcf,-zcmop,-zcmp,-zcmt,-zdinx,-zfa,-zfbfmin,-zfh,-zfhmin,-zfinx,-zhinx,-zhinxmin,-zic64b,-zicbom,-zicbop,-zicboz,-ziccamoa,-ziccif,-zicclsm,-ziccrse,-zicntr,-zicond,-zihintntl,-zihintpause,-zihpm,-zimop,-zk,-zkn,-zknd,-zkne,-zknh,-zkr,-zks,-zksed,-zksh,-zkt,-ztso,-zvbb,-zvbc,-zvfbfmin,-zvfbfwma,-zvfh,-zvfhmin,-zvkb,-zvkg,-zvkn,-zvknc,-zvkned,-zvkng,-zvknha,-zvknhb,-zvks,-zvksc,-zvksed,-zvksg,-zvksh,-zvkt,-zvl1024b,-zvl16384b,-zvl2048b,-zvl256b,-zvl32768b,-zvl4096b,-zvl512b,-zvl65536b,-zvl8192b" }
```

Command/backtrace:
```
> /scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt --passes loop-vectorize reduced.ll -S
opt: /scratch/tc-testing/tc-compiler-fuzz-trunk/llvm/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:7394: llvm::VectorizationFactor llvm::LoopVectorizationPlanner::computeBestVF(): Assertion `(BestFactor.Width == LegacyVF.Width || planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width), CostCtx, OrigLoop)) && " VPlan cost model and legacy cost model disagreed"' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.      Program arguments: /scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt --passes loop-vectorize reduced.ll -S
1.      Running pass "function(loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>)" on module "reduced.ll"
2.      Running pass "loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>" on function "g"
 #0 0x00005fabff5a9d80 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x3072d80)
 #1 0x00005fabff5a718f llvm::sys::RunSignalHandlers() (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x307018f)
 #2 0x00005fabff5a72e5 SignalHandler(int) Signals.cpp:0:0
 #3 0x00007fe3d2042520 (/lib/x86_64-linux-gnu/libc.so.6+0x42520)
 #4 0x00007fe3d20969fc __pthread_kill_implementation ./nptl/pthread_kill.c:44:76
 #5 0x00007fe3d20969fc __pthread_kill_internal ./nptl/pthread_kill.c:78:10
 #6 0x00007fe3d20969fc pthread_kill ./nptl/pthread_kill.c:89:10
 #7 0x00007fe3d2042476 gsignal ./signal/../sysdeps/posix/raise.c:27:6
 #8 0x00007fe3d20287f3 abort ./stdlib/abort.c:81:7
 #9 0x00007fe3d202871b _nl_load_domain ./intl/loadmsgcat.c:1177:9
#10 0x00007fe3d2039e96 (/lib/x86_64-linux-gnu/libc.so.6+0x39e96)
#11 0x00005fabfe550a26 llvm::LoopVectorizationPlanner::computeBestVF() (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x2019a26)
#12 0x00005fabfe5678fb llvm::LoopVectorizePass::processLoop(llvm::Loop*) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x20308fb)
#13 0x00005fabfe56a911 llvm::LoopVectorizePass::runImpl(llvm::Function&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x2033911)
#14 0x00005fabfe56af87 llvm::LoopVectorizePass::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x2033f87)
#15 0x00005fabfd3adf96 llvm::detail::PassModel<llvm::Function, llvm::LoopVectorizePass, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0xe76f96)
#16 0x00005fabff3b51ef llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x2e7e1ef)
#17 0x00005fabfd3b11c6 llvm::detail::PassModel<llvm::Function, llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0xe7a1c6)
#18 0x00005fabff3b357b llvm::ModuleToFunctionPassAdaptor::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x2e7c57b)
#19 0x00005fabfd3ad376 llvm::detail::PassModel<llvm::Module, llvm::ModuleToFunctionPassAdaptor, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0xe76376)
#20 0x00005fabff3b40dd llvm::PassManager<llvm::Module, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x2e7d0dd)
#21 0x00005fabfcbd49db llvm::runPassPipeline(llvm::StringRef, llvm::Module&, llvm::TargetMachine*, llvm::TargetLibraryInfoImpl*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::StringRef, llvm::ArrayRef<llvm::PassPlugin>, llvm::ArrayRef<std::function<void (llvm::PassBuilder&)>>, llvm::opt_tool::OutputKind, llvm::opt_tool::VerifierKind, bool, bool, bool, bool, bool, bool, bool) (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x69d9db)
#22 0x00005fabfcbc6701 optMain (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x68f701)
#23 0x00007fe3d2029d90 __libc_start_call_main ./csu/../sysdeps/nptl/libc_start_call_main.h:58:16
#24 0x00007fe3d2029e40 call_init ./csu/../csu/libc-start.c:128:20
#25 0x00007fe3d2029e40 __libc_start_main ./csu/../csu/libc-start.c:379:5
#26 0x00005fabfcbbcc65 _start (/scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt+0x685c65)
zsh: IOT instruction (core dumped)  /scratch/tc-testing/tc-compiler-fuzz-trunk/build-gcv/build-llvm-linux/bin/opt
```
https://godbolt.org/z/TTchhsf9h

Found via fuzzer

Related/prior issue: https://github.com/llvm/llvm-project/issues/107473

cc @fhahn