Track SIMDPirates vf(n)m(add/sub)231 use.

Author: chriselrod

src/LoopVectorization.jl

@@ -8,6 +8,7 @@ using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector
     PackedStridedPointer, SparseStridedPointer, RowMajorStridedPointer, StaticStridedPointer, StaticStridedStruct
 using SIMDPirates: VECTOR_SYMBOLS, evadd, evmul, vrange, reduced_add, reduced_prod, reduce_to_add, reduce_to_prod,
     sizeequivalentfloat, sizeequivalentint, vadd!, vsub!, vmul!, vfdiv!, vfmadd!, vfnmadd!, vfmsub!, vfnmsub!,
+    vfmadd231, vfmsub231, vfnmadd231, vfnmsub231,
     vmullog2, vmullog10, vdivlog2, vdivlog10, vmullog2add!, vmullog10add!, vdivlog2add!, vdivlog10add!, vfmaddaddone
 using Base.Broadcast: Broadcasted, DefaultArrayStyle
 using LinearAlgebra: Adjoint, Transpose

src/broadcast.jl

@@ -86,7 +86,7 @@ function add_broadcast!(
     setC.reduced_children = kvec
     # compute Cₘₙ += Aₘₖ * Bₖₙ
     reductop = Operation(
-        ls, mC, elementbytes, :vmuladd, compute, reductdeps, kvec, Operation[loadA, loadB, setC]
+        ls, mC, elementbytes, :vfmadd231, compute, reductdeps, kvec, Operation[loadA, loadB, setC]
     )
     reductop = pushop!(ls, reductop, mC)    
     reductfinal = Operation(

src/costs.jl

@@ -152,6 +152,10 @@ const COST = Dict{Instruction,InstructionCost}(
     Instruction(:vfmsub) => InstructionCost(4,0.5), # - and * will fuse into this, so much of the time they're not twice as expensive
     Instruction(:vfnmadd) => InstructionCost(4,0.5), # + and -* will fuse into this, so much of the time they're not twice as expensive
     Instruction(:vfnmsub) => InstructionCost(4,0.5), # - and -* will fuse into this, so much of the time they're not twice as expensive
+    Instruction(:vfmadd231) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
+    Instruction(:vfmsub231) => InstructionCost(4,0.5), # - and * will fuse into this, so much of the time they're not twice as expensive
+    Instruction(:vfnmadd231) => InstructionCost(4,0.5), # + and -* will fuse into this, so much of the time they're not twice as expensive
+    Instruction(:vfnmsub231) => InstructionCost(4,0.5), # - and -* will fuse into this, so much of the time they're not twice as expensive
     Instruction(:vfmadd!) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
     Instruction(:vfnmadd!) => InstructionCost(4,0.5), # + and -* will fuse into this, so much of the time they're not twice as expensive
     Instruction(:vfmsub!) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
@@ -233,6 +237,10 @@ const REDUCTION_CLASS = Dict{Symbol,Float64}(
     :vfmsub => ADDITIVE_IN_REDUCTIONS,
     :vfnmadd => ADDITIVE_IN_REDUCTIONS,
     :vfnmsub => ADDITIVE_IN_REDUCTIONS,
+    :vfmadd231 => ADDITIVE_IN_REDUCTIONS,
+    :vfmsub231 => ADDITIVE_IN_REDUCTIONS,
+    :vfnmadd231 => ADDITIVE_IN_REDUCTIONS,
+    :vfnmsub231 => ADDITIVE_IN_REDUCTIONS,
     :vfmadd! => ADDITIVE_IN_REDUCTIONS,
     :vfnmadd! => ADDITIVE_IN_REDUCTIONS,
     :vfmsub! => ADDITIVE_IN_REDUCTIONS,
@@ -316,6 +324,10 @@ const FUNCTIONSYMBOLS = Dict{Type{<:Function},Instruction}(
     typeof(SIMDPirates.vfmsub) => :vfmsub,
     typeof(SIMDPirates.vfnmadd) => :vfnmadd,
     typeof(SIMDPirates.vfnmsub) => :vfnmsub,
+    typeof(SIMDPirates.vfmadd231) => :vfmadd231,
+    typeof(SIMDPirates.vfmsub231) => :vfmsub231,
+    typeof(SIMDPirates.vfnmadd231) => :vfnmadd231,
+    typeof(SIMDPirates.vfnmsub231) => :vfnmsub231,
     typeof(SIMDPirates.vfmadd!) => :vfmadd!,
     typeof(SIMDPirates.vfnmadd!) => :vfnmadd!,
     typeof(SIMDPirates.vfmsub!) => :vfmsub!,