A few changes to determinestrategy, and update benchmark results.

Author: chriselrod

Manifest.toml

@@ -42,9 +42,9 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [[SIMDPirates]]
 deps = ["VectorizationBase"]
-git-tree-sha1 = "9fc6737cd40087e7d486f8e81fb5be8ad18f970b"
+git-tree-sha1 = "1a9cbe1be1f5d43ac49eeb38ca64dd78e50a0cc6"
 uuid = "21efa798-c60a-11e8-04d3-e1a92915a26a"
-version = "0.7.2"
+version = "0.7.3"
 
 [[SLEEFPirates]]
 deps = ["Libdl", "SIMDPirates", "VectorizationBase"]
@@ -69,6 +69,6 @@ version = "0.1.0"
 
 [[VectorizationBase]]
 deps = ["CpuId", "LinearAlgebra"]
-git-tree-sha1 = "76e8817f7732d9a127191f5bcd5fe3a5eed0fb3e"
+git-tree-sha1 = "83f073a514b5d654cc9c72ae283a33388a0d0386"
 uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
-version = "0.9.2"
+version = "0.9.3"

Project.toml

@@ -13,10 +13,10 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [compat]
 OffsetArrays = "1"
-SIMDPirates = "0.7.1"
+SIMDPirates = "0.7.3"
 SLEEFPirates = "0.4"
 UnPack = "0"
-VectorizationBase = "0.9.2"
+VectorizationBase = "0.9.3"
 julia = "1.1"
 
 [extras]

docs/src/assets/bench_AmulB_v1.svg

@@ -352,16 +352,25 @@ function solve_tilesize(
     reg_pressure::AbstractVector{Float64},
     W::Int, vectorized::Symbol
 )
-    maxTbase = maxUbase = 4#8
+    maxTbase = maxUbase = 6#8
     maxT = maxTbase#8
     maxU = maxUbase#8
     tiledloop = getloop(ls, tiled)
     unrolledloop = getloop(ls, unrolled)
     if isstaticloop(tiledloop)
+        if length(tiledloop) ≤ 4
+            T = length(tiledloop)
+            U = max(1, solve_tilesize_constT(cost_vec, reg_pressure, T))
+            return U, T, tile_cost(cost_vec, U, T, length(unrolledloop), T)
+        end
         maxT = min(4maxT, length(tiledloop))
     end
     if isstaticloop(unrolledloop)
         UL = length(unrolledloop)
+        if unrolled !== vectorized && UL ≤ 4
+            T = max(1, solve_tilesize_constU(cost_vec, reg_pressure, UL))
+            return UL, T, tile_cost(cost_vec, UL, T, UL, length(tiledloop))
+        end
         UL = unrolled === vectorized ? cld(UL,W) : UL
         maxU = min(4maxU, UL)
     end
@@ -383,11 +392,22 @@ function set_upstream_family!(adal::Vector{T}, op::Operation, val::T) where {T}
         set_upstream_family!(adal, opp, val)
     end
 end
-
+function stride_penalty_opdependent(ls::LoopSet, op::Operation, order::Vector{Symbol}, contigsym::Symbol)
+    num_loops = length(order)
+    firstloopdeps = loopdependencies(findparent(ls, contigsym))
+    iter = 1
+    for i ∈ 0:num_loops - 1
+        loopsym = order[num_loops - i]
+        loopsym ∈ firstloopdeps && return iter
+        iter *= length(getloop(ls, loopsym))
+    end
+    iter
+end
 function stride_penalty(ls::LoopSet, op::Operation, order::Vector{Symbol})
     num_loops = length(order)
-    contigsym = first(loopdependencies(op))
+    contigsym = first(loopdependencies(op.ref))
     contigsym == Symbol("##DISCONTIGUOUSSUBARRAY##") && return 0
+    first(op.ref.loopedindex) || return stride_penalty_opdependent(ls, op, order, contigsym)
     iter = 1
     for i ∈ 0:num_loops - 1
         loopsym = order[num_loops - i]
@@ -405,19 +425,40 @@ function stride_penalty(ls::LoopSet, order::Vector{Symbol})
     end
     stridepenalty# * 1e-9
 end
-function convolution_cost_factor(ls::LoopSet, op::Operation, u1::Symbol, u2::Symbol)
-    (u1 ∈ loopdependencies(op) && u2 ∈ loopdependencies(op)) || return 1.0, 1.0
+function isoptranslation(ls::LoopSet, op::Operation, u1::Symbol, u2::Symbol, vectorized::Symbol)
+    (vectorized == u1 || vectorized == u2) && return false, false
+    (u1 ∈ loopdependencies(op) && u2 ∈ loopdependencies(op)) || return false, false
     istranslation = false
     inds = getindices(op); li = op.ref.loopedindex
+    translationplus = false
     for i ∈ eachindex(li)
         if !li[i]
             opp = findparent(ls, inds[i + (first(inds) === Symbol("##DISCONTIGUOUSSUBARRAY##"))])
             if instruction(opp).instr ∈ (:+, :-) && u1 ∈ loopdependencies(opp) && u2 ∈ loopdependencies(opp)
                 istranslation = true
+                translationplus = instruction(opp).instr === :+
+            end
+        end
+    end
+    istranslation, translationplus
+end
+function convolution_cost_factor(ls::LoopSet, op::Operation, u1::Symbol, u2::Symbol, v::Symbol)
+    if first(isoptranslation(ls, op, u1, u2, v))
+        for loop ∈ ls.loops
+            # If another loop is short, assume that LLVM will unroll it, in which case
+            # we want to be a little more conservative in terms of register pressure.
+            #FIXME: heuristic hack to get some desired behavior.
+            if isstaticloop(loop) && length(loop) ≤ 4
+                itersym = loop.itersymbol
+                if itersym !== u1 && itersym !== u2
+                    return (0.25, 1.0)
+                end
             end
         end
+        (0.25, 0.5)
+    else
+        (1.0, 1.0)
     end
-    istranslation ? (0.25, 1.0) : (1.0, 1.0)
 end
 # Just tile outer two loops?
 # But optimal order within tile must still be determined
@@ -484,7 +525,7 @@ function evaluate_cost_tile(
         istiled = unrolledtiled[2,id]
         rt, lat, rp = cost(ls, op, vectorized, Wshift, size_T)
         if isload(op)
-            factor1, factor2 = convolution_cost_factor(ls, op, unrolled, tiled)
+            factor1, factor2 = convolution_cost_factor(ls, op, unrolled, tiled, vectorized)
             rt *= factor1; rp *= factor2;
         end
         # @show op rt, lat, rp

docs/src/assets/bench_AmulBt_v1.svg

@@ -1,38 +1,38 @@
-function pushvectorload!(q::Expr, op::Operation, var::Symbol, td::UnrollArgs, U::Int, W::Symbol, mask, vecnotunrolled::Bool)
-    @unpack u, unrolled = td
-    ptr = refname(op)
-    name, mo = name_memoffset(var, op, td, W, vecnotunrolled)
-    instrcall = Expr(:call, lv(:vload), ptr, mo)
-    if mask !== nothing && (vecnotunrolled || u == U - 1)
-        push!(instrcall.args, mask)
-    end
-    push!(q.args, Expr(:(=), name, instrcall))
-end
 function lower_load_scalar!(
     q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
-    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
+    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing, umin::Int = 0
 )
     loopdeps = loopdependencies(op)
     @assert vectorized ∉ loopdeps
     var = variable_name(op, suffix)
     ptr = refname(op)
     isunrolled = unrolled ∈ loopdeps
     U = isunrolled ? U : 1
-    for u ∈ 0:U-1
+    for u ∈ umin:U-1
         varname = varassignname(var, u, isunrolled)
         td = UnrollArgs(u, unrolled, tiled, suffix)
         push!(q.args, Expr(:(=), varname, Expr(:call, lv(:vload), ptr, mem_offset_u(op, td))))
     end
     nothing
 end
+function pushvectorload!(q::Expr, op::Operation, var::Symbol, td::UnrollArgs, U::Int, W::Symbol, mask, vecnotunrolled::Bool)
+    @unpack u, unrolled = td
+    ptr = refname(op)
+    name, mo = name_memoffset(var, op, td, W, vecnotunrolled)
+    instrcall = Expr(:call, lv(:vload), ptr, mo)
+    if mask !== nothing && (vecnotunrolled || u == U - 1)
+        push!(instrcall.args, mask)
+    end
+    push!(q.args, Expr(:(=), name, instrcall))
+end
 function lower_load_vectorized!(
     q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
-    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
+    suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing, umin::Int = 0
 )
     loopdeps = loopdependencies(op)
     @assert vectorized ∈ loopdeps
     if unrolled ∈ loopdeps
-        umin = 0
+        umin = umin
         U = U
     else
         umin = -1
@@ -51,12 +51,28 @@ end
 # TODO: this code should be rewritten to be more "orthogonal", so that we're just combining separate pieces.
 # Using sentinel values (eg, T = -1 for non tiling) in part to avoid recompilation.
 function lower_load!(
-    q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
+    q::Expr, op::Operation, vectorized::Symbol, ls::LoopSet, unrolled::Symbol, tiled::Symbol, U::Int,
     suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
 )
+    if !isnothing(suffix) && suffix > 0
+        istr, ispl = isoptranslation(ls, op, unrolled, tiled, vectorized)
+        if istr && ispl
+            varnew = variable_name(op, suffix)
+            varold = variable_name(op, suffix - 1)
+            for u ∈ 0:U-2
+                push!(q.args, Expr(:(=), Symbol(varnew, u), Symbol(varold, u + 1)))
+            end
+            umin = U - 1
+        else
+            umin = 0
+        end
+    else
+        umin = 0
+    end
+    W = ls.W
     if vectorized ∈ loopdependencies(op)
-        lower_load_vectorized!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+        lower_load_vectorized!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask, umin)
     else
-        lower_load_scalar!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+        lower_load_scalar!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask, umin)
     end
 end

docs/src/assets/bench_Amulvb_v1.svg

@@ -21,7 +21,7 @@ function lower!(
             lower_zero!(q, op, vectorized, ls, unrolled, U, suffix, zerotyp)
         end
     elseif isload(op)
-        lower_load!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+        lower_load!(q, op, vectorized, ls, unrolled, tiled, U, suffix, mask)
     elseif iscompute(op)
         lower_compute!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
     elseif isstore(op)
@@ -47,7 +47,7 @@ function lower!(
                 lower_zero!(q, op, vectorized, ls, unrolled, U, suffix, zerotyp)
             end
         elseif isload(op)
-            lower_load!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
+            lower_load!(q, op, vectorized, ls, unrolled, tiled, U, suffix, mask)
         elseif iscompute(op)
             lower_compute!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
         end

docs/src/assets/bench_AplusAt_v1.svg

@@ -1,17 +1,19 @@
-function Loop(ls::LoopSet, l::Int, sym::Symbol, ::Type{UnitRange{Int}})
-    start = gensym(String(sym)*"_loopstart"); stop = gensym(String(sym)*"_loopstop")
-    pushpreamble!(ls, Expr(:(=), start, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:(.), Expr(:ref, :lb, l), QuoteNode(:start)))))
-    pushpreamble!(ls, Expr(:(=), stop, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:(.), Expr(:ref, :lb, l), QuoteNode(:stop)))))
+function Loop(ls::LoopSet, ex::Expr, sym::Symbol, ::Type{<:AbstractUnitRange})
+    ssym = String(sym)
+    start = gensym(ssym*"_loopstart"); stop = gensym(ssym*"_loopstop"); loopsym = gensym(ssym * "_loop")
+    pushpreamble!(ls, Expr(:(=), loopsym, ex))
+    pushpreamble!(ls, Expr(:(=), start, Expr(:call, :first, loopsym)))
+    pushpreamble!(ls, Expr(:(=), stop, Expr(:call, :last, loopsym)))
     Loop(sym, 0, 1024, start, stop, false, false)::Loop
 end
-function Loop(ls::LoopSet, l::Int, sym::Symbol, ::Type{StaticUpperUnitRange{U}}) where {U}
+function Loop(ls::LoopSet, ex::Expr, sym::Symbol, ::Type{StaticUpperUnitRange{U}}) where {U}
     start = gensym(String(sym)*"_loopstart")
-    pushpreamble!(ls, Expr(:(=), start, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:(.), Expr(:ref, :lb, l), QuoteNode(:L)))))
+    pushpreamble!(ls, Expr(:(=), start, Expr(:(.), ex, QuoteNode(:L))))
     Loop(sym, U - 1024, U, start, Symbol(""), false, true)::Loop
 end
-function Loop(ls::LoopSet, l::Int, sym::Symbol, ::Type{StaticLowerUnitRange{L}}) where {L}
+function Loop(ls::LoopSet, ex::Expr, sym::String, ::Type{StaticLowerUnitRange{L}}) where {L}
     stop = gensym(String(sym)*"_loopstop")
-    pushpreamble!(ls, Expr(:(=), stop, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:(.), Expr(:ref, :lb, l), QuoteNode(:U)))))
+    pushpreamble!(ls, Expr(:(=), stop, Expr(:(.), ex, QuoteNode(:U))))
     Loop(sym, L, L + 1024, Symbol(""), stop, true, false)::Loop
 end
 # Is there any likely way to generate such a range?
@@ -21,17 +23,12 @@ end
 #     pushpreamble!(ls, Expr(:(=), stop, Expr(:call, :(+), start, N - 1)))
 #     Loop(gensym(:n), 0, N, start, stop, false, false)::Loop
 # end
-function Loop(ls, l, sym::Symbol, ::Type{StaticUnitRange{L,U}}) where {L,U}
+function Loop(::LoopSet, ::Expr, sym::Symbol, ::Type{StaticUnitRange{L,U}}) where {L,U}
     Loop(sym, L, U, Symbol(""), Symbol(""), true, true)::Loop
 end
 
-function Loop(ls::LoopSet, l::Int, k::Int, sym::Symbol, ::Type{<:CartesianIndices{N}}) where N
-    str = String(sym)*'#'*string(k)*'#'
-    start = gensym(str*"_loopstart"); stop = gensym(str*"_loopstop")
-    axisexpr = Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:ref, Expr(:., Expr(:ref, :lb, l), QuoteNode(:indices)), k))
-    pushpreamble!(ls, Expr(:(=), start, Expr(:call, :first, axisexpr)))
-    pushpreamble!(ls, Expr(:(=), stop, Expr(:call, :last, axisexpr)))
-    Loop(Symbol(str), 0, 1024, start, stop, false, false)::Loop
+function extract_loop(l)
+    Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:ref, :lb, l))
 end
 
 function add_loops!(ls::LoopSet, LPSYM, LB)
@@ -41,14 +38,16 @@ function add_loops!(ls::LoopSet, LPSYM, LB)
         if l<:CartesianIndices
             add_loops!(ls, i, sym, l)
         else
-            add_loop!(ls, Loop(ls, i, sym, l)::Loop)
+            add_loop!(ls, Loop(ls, extract_loop(i), sym, l)::Loop)
             push!(ls.loopsymbol_offsets, ls.loopsymbol_offsets[end]+1)
         end
     end
 end
-function add_loops!(ls, i, sym, l::Type{<:CartesianIndices{N}}) where N
+function add_loops!(ls::LoopSet, i::Int, sym::Symbol, l::Type{CartesianIndices{N,T}}) where {N,T}
+    ssym = String(sym)
     for k = N:-1:1
-        add_loop!(ls, Loop(ls, i, k, sym, l)::Loop)
+        axisexpr = Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:ref, Expr(:., Expr(:ref, :lb, i), QuoteNode(:indices)), k))
+        add_loop!(ls, Loop(ls, axisexpr, Symbol(ssym*'#'*string(k)*'#'), T.parameters[k])::Loop)
     end
     push!(ls.loopsymbol_offsets, ls.loopsymbol_offsets[end]+N)
 end

docs/src/assets/bench_AtmulB_v1.svg

@@ -1,6 +1,6 @@
 @testset "GEMM" begin
     # using LoopVectorization, LinearAlgebra, Test; T = Float64
-    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 4)
+    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (5, 5)
     AmulBtq1 = :(for m ∈ 1:size(A,1), n ∈ 1:size(B,2)
                  C[m,n] = zeroB
                  for k ∈ 1:size(A,2)
@@ -291,7 +291,11 @@
                C[m,n] = Cₘₙ
                end)
     lsr2amb = LoopVectorization.LoopSet(r2ambq);
-    @test LoopVectorization.choose_order(lsr2amb) == ([:n, :m, :k], :k, :n, :m, Unum & -2, Tnum)
+    if LoopVectorization.VectorizationBase.REGISTER_COUNT == 32
+        @test LoopVectorization.choose_order(lsr2amb) == ([:n, :m, :k], :k, :n, :m, 3, 6)
+    else
+        @test LoopVectorization.choose_order(lsr2amb) == ([:n, :m, :k], :k, :n, :m, 2, 4)
+    end
     function rank2AmulBavx!(C, Aₘ, Aₖ, B)
         @avx for m ∈ 1:size(C,1), n ∈ 1:size(C,2)
             Cₘₙ = zero(eltype(C))

docs/src/assets/bench_AtmulBt_v1.svg

@@ -2,7 +2,7 @@ using LoopVectorization
 using Test
 
 @testset "GEMV" begin
-    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 4)
+    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 6)
     gemvq = :(for i ∈ eachindex(y)
               yᵢ = 0.0
               for j ∈ eachindex(x)

docs/src/assets/bench_Atmulvb_v1.svg

@@ -1,6 +1,6 @@
 
 @testset "Miscellaneous" begin
-    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 4)
+    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 6)
     dot3q = :(for m ∈ 1:M, n ∈ 1:N
               s += x[m] * A[m,n] * y[n]
               end);