Update benchmark plots, make a few changes to improve handling of aliasing, and quit using masktable now that mask in VectorizationBase v0.9 takes a length argument and returns an all-on mask when the remainder is 0.

Author: chriselrod

docs/src/assets/bench_AmulB_v1.svg

@@ -213,7 +213,7 @@ end
         LHS = ind === nothing ? gensym() : vptrs[ind]
         assigned_names[i] = LHS
         d = (D[i])::Union{Nothing,Int}
-        if d === nothing # stridedpointer instead of noaliasstridedpointer, because alias info will be lost across function boundary...
+        if d === nothing
             num_arrays += 1
             RHS = Expr(:call, lv(:stridedpointer), Expr(:ref, :vargs, ari), Expr(:ref, :arraydescript, ari))
         else #subsetview
@@ -255,7 +255,7 @@ function generate_call(ls::LoopSet, IUT, debug::Bool = false)
         )
         debug && deleteat!(q.args, 2)
         foreach(ref -> push!(q.args, vptr(ref)), ls.refs_aliasing_syms)
-    else
+    else# not forcing inline; calling __avx__! which calls an inlined _avx_!
         arraydescript = Expr(:tuple)
         q = Expr(
             :call, lv(:__avx__!), Expr(:call, Expr(:curly, :Val, (U,T))),
@@ -288,7 +288,7 @@ function setup_call_noinline(ls::LoopSet, U = zero(Int8), T = zero(Int8))
         if ex isa Expr && ex.head === :(=) && length(ex.args) == 2
             if ex.args[2] isa Expr && ex.args[2].head === :call
                 gr = first(ex.args[2].args)
-                if gr == lv(:noaliasstridedpointer)
+                if gr == lv(:stridedpointer)
                     array = ex.args[2].args[2]
                     arrayid = findfirst(a -> a === array, ls.includedactualarrays)
                     if arrayid isa Int

docs/src/assets/bench_AmulBt_v1.svg

@@ -57,12 +57,18 @@ function lower_compute!(
     # parentsyms = [opp.variable for opp ∈ parents(op)]
     Uiter = opunrolled ? U - 1 : 0
     isreduct = isreduction(op)
-    if !isnothing(suffix) && isreduct && tiledouterreduction == -1
-        instrfid = findfirst(isequal(instr.instr), (:vfmadd, :vfnmadd, :vfmsub, :vfnmsub))
-        if instrfid !== nothing
+    # if instr.instr === :vfmadd_fast
+        # diffdeps = !any(opp -> isload(opp) && all(in(loopdependencies(opp)), loopdependencies(op)), parents(op)) # want to instcombine when parent load's deps are superset
+        # @show suffix, !isnothing(suffix), isreduct, diffdeps
+    # end
+    if !isnothing(suffix) && isreduct
+        # instrfid = findfirst(isequal(instr.instr), (:vfmadd, :vfnmadd, :vfmsub, :vfnmsub))
+        instrfid = findfirst(isequal(instr.instr), (:vfmadd_fast, :vfnmadd_fast, :vfmsub_fast, :vfnmsub_fast))
+        if instrfid !== nothing && !any(opp -> isload(opp) && all(in(loopdependencies(opp)), loopdependencies(op)), parents(op)) # want to instcombine when parent load's deps are superset
             instr = Instruction((:vfmadd231, :vfnmadd231, :vfmsub231, :vfnmsub231)[instrfid])
         end
     end
+    # @show instr.instr
     maskreduct = mask !== nothing && isreduct && vectorized ∈ reduceddependencies(op) #any(opp -> opp.variable === var, parents_op)
     # if a parent is not unrolled, the compiler should handle broadcasting CSE.
     # because unrolled/tiled parents result in an unrolled/tiled dependendency,

docs/src/assets/bench_Amulvb_v1.svg

@@ -1,3 +1,6 @@
+using VectorizationBase: vnoaliasstore!
+const STOREOP = :vnoaliasstore!
+# const STOREOP = :vstore!
 variable_name(op::Operation, ::Nothing) = mangledvar(op)
 variable_name(op::Operation, suffix) = Symbol(mangledvar(op), suffix, :_)
 function reduce_range!(q::Expr, toreduct::Symbol, instr::Instruction, Uh::Int, Uh2::Int)
@@ -66,7 +69,7 @@ function lower_conditionalstore_scalar!(
         varname = varassignname(var, u, parentisunrolled)
         condvarname = varassignname(condvar, u, condunrolled)
         td = UnrollArgs(u, unrolled, tiled, suffix)
-        push!(q.args, Expr(:&&, condvarname, Expr(:call, lv(:vstore!), ptr, varname, mem_offset_u(op, td))))
+        push!(q.args, Expr(:&&, condvarname, Expr(:call, lv(STOREOP), ptr, varname, mem_offset_u(op, td))))
     end
     nothing
 end
@@ -99,7 +102,7 @@ function lower_conditionalstore_vectorized!(
         td = UnrollArgs(u, unrolled, tiled, suffix)
         name, mo = name_memoffset(var, op, td, W, vecnotunrolled, parentisunrolled)
         condvarname = varassignname(condvar, u, condunrolled)
-        instrcall = Expr(:call, lv(:vstore!), ptr, name, mo)
+        instrcall = Expr(:call, lv(STOREOP), ptr, name, mo)
         if mask !== nothing && (vecnotunrolled || u == U - 1)
             push!(instrcall.args, Expr(:call, :&, condvarname, mask))
         else
@@ -119,7 +122,7 @@ function lower_store_scalar!(
     for u ∈ 0:U-1
         varname = varassignname(var, u, parentisunrolled)
         td = UnrollArgs(u, unrolled, tiled, suffix)
-        push!(q.args, Expr(:call, lv(:vstore!), ptr, varname, mem_offset_u(op, td)))
+        push!(q.args, Expr(:call, lv(STOREOP), ptr, varname, mem_offset_u(op, td)))
     end
     nothing
 end
@@ -143,7 +146,7 @@ function lower_store_vectorized!(
     for u ∈ umin:U-1
         td = UnrollArgs(u, unrolled, tiled, suffix)
         name, mo = name_memoffset(var, op, td, W, vecnotunrolled, parentisunrolled)
-        instrcall = Expr(:call, lv(:vstore!), ptr, name, mo)
+        instrcall = Expr(:call, lv(STOREOP), ptr, name, mo)
         if mask !== nothing && (vecnotunrolled || u == U - 1)
             push!(instrcall.args, mask)
         end

docs/src/assets/bench_AplusAt_v1.svg

@@ -10,7 +10,7 @@
 
 function lower!(
     q::Expr, op::Operation, vectorized::Symbol, ls::LoopSet, unrolled::Symbol, tiled::Symbol, U::Int,
-    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned}
+    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned}, ::Nothing
 )
     W = ls.W
     if isconstant(op)
@@ -29,11 +29,35 @@ function lower!(
     # elseif isloopvalue(op)
     end
 end
+function lower!(
+    q::Expr, op::Operation, vectorized::Symbol, ls::LoopSet, unrolled::Symbol, tiled::Symbol, U::Int,
+    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned}, filterstore::Bool
+)
+    W = ls.W
+    if filterstore
+        if isstore(op)
+            lower_store!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+        end
+    else
+        if isconstant(op)
+            zerotyp = zerotype(ls, op)
+            if zerotyp == INVALID
+                lower_constant!(q, op, vectorized, ls, unrolled, U, suffix)
+            else
+                lower_zero!(q, op, vectorized, ls, unrolled, U, suffix, zerotyp)
+            end
+        elseif isload(op)
+            lower_load!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+        elseif iscompute(op)
+            lower_compute!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+        end
+    end
+end
 function lower!(
     q::Expr, ops::AbstractVector{Operation}, vectorized::Symbol, ls::LoopSet, unrolled::Symbol, tiled::Symbol, U::Int,
-    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned}
+    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned}, filterstore = nothing
 )
-    foreach(op -> lower!(q, op, vectorized, ls, unrolled, tiled, U, suffix, mask), ops)
+    foreach(op -> lower!(q, op, vectorized, ls, unrolled, tiled, U, suffix, mask, filterstore), ops)
 end
 
 function lower_block(ls::LoopSet, us::UnrollSpecification, n::Int, inclmask::Bool, UF)
@@ -62,26 +86,36 @@ function lower_block(
         lower!(blockq, ops[2,1,prepost,n], vectorized, ls, unrolled, tiled, U, nothing, mask)
         # end
         if length(ops[1,2,prepost,n]) + length(ops[2,2,prepost,n]) > 0
-            for t ∈ 0:T-1
-                if t == 0
-                    push!(blockq.args, Expr(:(=), tiled, tiledsym(tiled)))
-                elseif tiledloopnum == vectorizedloopnum
-                    push!(blockq.args, Expr(:(=), tiled, Expr(:call, lv(:valadd), ls.W, tiled)))
-                else
-                    push!(blockq.args, Expr(:+=, tiled, 1))
-                end
-                # !U &&  T
-                if dontmaskfirsttiles && t < T - 1
-                    lower!(blockq, ops[1,2,prepost,n], vectorized, ls, unrolled, tiled, U, t, nothing)
-                    # for u ∈ 0:U-1 #  U &&  T
-                    lower!(blockq, ops[2,2,prepost,n], vectorized, ls, unrolled, tiled, U, t, nothing)
-                    # end
-                else
-                    lower!(blockq, ops[1,2,prepost,n], vectorized, ls, unrolled, tiled, U, t, mask)
-                    # for u ∈ 0:U-1 #  U &&  T
-                    lower!(blockq, ops[2,2,prepost,n], vectorized, ls, unrolled, tiled, U, t, mask)
-                    # end
+            for store ∈ (false,true)
+                # let store = nothing
+                nstores = 0
+                opsv1 = ops[1,2,prepost,n]
+                opsv2 = ops[2,2,prepost,n]
+                iszero(length(opsv1) + length(opsv2)) && continue
+                iszero(length(opsv1)) || (nstores += sum(isstore, opsv1))
+                iszero(length(opsv2)) || (nstores += sum(isstore, opsv2))
+                for t ∈ 0:T-1
+                    if t == 0
+                        push!(blockq.args, Expr(:(=), tiled, tiledsym(tiled)))
+                    elseif tiledloopnum == vectorizedloopnum
+                        push!(blockq.args, Expr(:(=), tiled, Expr(:call, lv(:valadd), ls.W, tiled)))
+                    else
+                        push!(blockq.args, Expr(:+=, tiled, 1))
+                    end
+                    # !U &&  T
+                    if dontmaskfirsttiles && t < T - 1
+                        lower!(blockq, opsv1, vectorized, ls, unrolled, tiled, U, t, nothing, store)
+                        # for u ∈ 0:U-1 #  U &&  T
+                        lower!(blockq, opsv2, vectorized, ls, unrolled, tiled, U, t, nothing, store)
+                        # end
+                    else
+                        lower!(blockq, opsv1, vectorized, ls, unrolled, tiled, U, t, mask, store)
+                        # for u ∈ 0:U-1 #  U &&  T
+                        lower!(blockq, opsv2, vectorized, ls, unrolled, tiled, U, t, mask, store)
+                        # end
+                    end
                 end
+                nstores == 0 && break
             end
         end
         if n > 1 && prepost == 1
@@ -276,8 +310,9 @@ function init_remblock(unrolledloop::Loop, unrolled::Symbol = unrolledloop.iters
 end
 
 function maskexpr(W::Symbol, looplimit)
-    rem = Expr(:call, lv(:valrem), W, looplimit)
-    Expr(:(=), Symbol("##mask##"), Expr(:call, lv(:masktable), W, rem))
+    Expr(:(=), Symbol("##mask##"), Expr(:call, lv(:mask), W, looplimit))
+    # rem = Expr(:call, lv(:valrem), W, looplimit)
+    # Expr(:(=), Symbol("##mask##"), Expr(:call, lv(:masktable), W, rem))
 end
 function definemask(loop::Loop, W::Symbol)
     if isstaticloop(loop)

docs/src/assets/bench_AtmulB_v1.svg

@@ -23,8 +23,8 @@ function add_vptr!(ls::LoopSet, array::Symbol, vptrarray::Symbol = vptr(array),
         if broadcast
             pushpreamble!(ls, Expr(:(=), vptrarray, Expr(:call, lv(:stridedpointer_for_broadcast), array)))
         else
-            # pushpreamble!(ls, Expr(:(=), vptrarray, Expr(:call, lv(:stridedpointer), array)))
-            pushpreamble!(ls, Expr(:(=), vptrarray, Expr(:call, lv(:noaliasstridedpointer), array)))
+            pushpreamble!(ls, Expr(:(=), vptrarray, Expr(:call, lv(:stridedpointer), array)))
+            # pushpreamble!(ls, Expr(:(=), vptrarray, Expr(:call, lv(:noaliasstridedpointer), array)))
         end
     end
     nothing

docs/src/assets/bench_AtmulBt_v1.svg

@@ -69,42 +69,46 @@ using Test
         @avx unroll=2 for i ∈ 1:length(x)
             z += x[i]*y[i]
         end
-        return z
+        z
     end
     function dot_unroll3avx(x::Vector{T}, y::Vector{T}) where {T<:Number}
         z = zero(T)
         @avx unroll=3 for i ∈ 1:length(x)
             z += x[i]*y[i]
         end
-        return z
+        z
     end
+    @macroexpand @avx inline=false unroll=2 for i ∈ 1:length(x)
+            z += x[i]*y[i]
+        end
+
     function dot_unroll2avx_noinline(x::Vector{T}, y::Vector{T}) where {T<:Number}
         z = zero(T)
-        @avx inline=false unroll=2 for i ∈ 1:length(x)
+        @avx inline=true unroll=2 for i ∈ 1:length(x)
             z += x[i]*y[i]
         end
-        return z
+        z
     end
     function dot_unroll3avx_inline(x::Vector{T}, y::Vector{T}) where {T<:Number}
         z = zero(T)
         @avx unroll=3 inline=true for i ∈ 1:length(x)
             z += x[i]*y[i]
         end
-        return z
+        z
     end
     function dot_unroll2_avx(x::Vector{T}, y::Vector{T}) where {T<:Number}
         z = zero(T)
         @_avx unroll=2 for i ∈ 1:length(x)
             z += x[i]*y[i]
         end
-        return z
+        z
     end
     function dot_unroll3_avx(x::Vector{T}, y::Vector{T}) where {T<:Number}
         z = zero(T)
         @_avx unroll=3 for i ∈ 1:length(x)
             z += x[i]*y[i]
         end
-        return z
+        z
     end
     function complex_dot_soa(
         xre::AbstractVector{T}, xim::AbstractVector{T},
@@ -116,13 +120,13 @@ using Test
             zre += xre[i]*yre[i] - xim[i]*yim[i]
             zim += xre[i]*yim[i] + xim[i]*yre[i]
         end
-        return Complex{T}(zre,zim)
+        Complex{T}(zre,zim)
     end
     qc = :(for i ∈ 1:length(xre)
            zre += xre[i]*yre[i] - xim[i]*yim[i]
            zim += xre[i]*yim[i] + xim[i]*yre[i]
            end);
-    lsc = LoopVectorization.LoopSet(qc)
+    lsc = LoopVectorization.LoopSet(qc);
     function complex_mul_with_index_offset!(c_re, c_im, a_re, a_im, b_re, b_im)
         @inbounds @simd ivdep for i = 1:length(a_re) - 1
             c_re[i] = b_re[i] * a_re[i + 1] - b_im[i] * a_im[i + 1]