Bump reqs (#96)

Author: chriselrod

Project.toml

@@ -13,11 +13,11 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [compat]
 ArrayInterface = "3.1.14"
-LoopVectorization = "0.12.32"
+LoopVectorization = "0.12.34"
 Static = "0.2"
 StrideArraysCore = "0.1.11"
 ThreadingUtilities = "0.4"
-VectorizationBase = "0.20.15"
+VectorizationBase = "0.20.16"
 julia = "1.6"
 
 [extras]

src/funcptrs.jl

@@ -25,8 +25,8 @@ end
 end
 call_loopmul!(C, A, B, α, β, M, K, N, ::Val{P}) where {P} = _call_loopmul!(C, A, B, α, β, M, K, N, Val{P}())
 
-struct SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,AP,BCP,ID,TT,W₁,W₂,R₁,R₂} <: Function end
-function (::SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,AP,BCP,ID,TT,W₁,W₂,R₁,R₂})(p::Ptr{UInt}) where {TC,TA,TB,Α,Β,Md,Kd,Nd,AP,BCP,ID,TT,W₁,W₂,R₁,R₂}
+struct SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,BCP,ID,TT,W₁,W₂,R₁,R₂} <: Function end
+function (::SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,BCP,ID,TT,W₁,W₂,R₁,R₂})(p::Ptr{UInt}) where {TC,TA,TB,Α,Β,Md,Kd,Nd,BCP,ID,TT,W₁,W₂,R₁,R₂}
     offset, C = load(p, TC, 2*sizeof(UInt))
     offset, A = load(p, TA, offset)
     offset, B = load(p, TB, offset)
@@ -35,7 +35,7 @@ function (::SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,AP,BCP,ID,TT,W₁,W₂,R₁,R₂
     offset, M = load(p, Md, offset)
     offset, K = load(p, Kd, offset)
     offset, N = load(p, Nd, offset)
-    offset, atomicp = load(p, AP, offset)
+    offset, atomicp = load(p, Ptr{UInt32}, offset)
     offset, bcachep = load(p, BCP, offset)
     offset, id = load(p, ID, offset)
     offset, total_ids = load(p, TT, offset)
@@ -66,9 +66,9 @@ end
 
 @inline function setup_syncmul!(
     p::Ptr{UInt}, C::TC, A::TA, B::TB, α::Α, β::Β, M::Md, K::Kd, N::Nd,
-    ap::AP,bcp::BCP,id::ID,tt::TT,::StaticFloat64{W₁},::StaticFloat64{W₂},::StaticFloat64{R₁},::StaticFloat64{R₂}
-) where {TC,TA,TB,Α,Β,Md,Kd,Nd,AP,BCP,ID,TT,W₁,W₂,R₁,R₂}
-    offset = store!(p, cfuncpointer(SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,AP,BCP,ID,TT,W₁,W₂,R₁,R₂}()), sizeof(UInt))
+    ap::Ptr{UInt32},bcp::BCP,id::ID,tt::TT,::StaticFloat64{W₁},::StaticFloat64{W₂},::StaticFloat64{R₁},::StaticFloat64{R₂}
+) where {TC,TA,TB,Α,Β,Md,Kd,Nd,BCP,ID,TT,W₁,W₂,R₁,R₂}
+    offset = store!(p, cfuncpointer(SyncMulFunc{TC,TA,TB,Α,Β,Md,Kd,Nd,BCP,ID,TT,W₁,W₂,R₁,R₂}()), sizeof(UInt))
     offset = store!(p, C, offset)
     offset = store!(p, A, offset)
     offset = store!(p, B, offset)

src/macrokernels.jl

@@ -116,15 +116,14 @@ end
 end
 @inline function alloc_a_pack(K, ::Val{T}) where {T}
   buffer = first_cache_buffer(Val(T))
+  alloc_a_pack(K, buffer), buffer
+end
+@inline alloc_a_pack(K, buffer::MemoryBuffer) = alloc_a_pack(K, align(pointer(buffer)))
+@inline function alloc_a_pack(K, bufferptr::Ptr{T}) where {T}
   mᵣ, nᵣ = matmul_params(Val(T))
   mᵣW = mᵣ * pick_vector_width(T)
-  bufferptr = if Sys.WORD_SIZE == 32
-    buffer
-  else
-    align(pointer(buffer))
-  end
   Apack = default_zerobased_stridedpointer(bufferptr, (One(), mᵣW, mᵣW * K)) # mᵣW x K x cld(M, mᵣW)
-  Apack, buffer
+  Apack
 end
 function packaloopmul!(
     C::AbstractStridedPointer{T},

src/matmul.jl

@@ -388,11 +388,11 @@ function matmul_pack_A_and_B!(
     mᵣW = mᵣ * W
     # atomicsync = Ref{NTuple{16,UInt}}()
     Mbsize, Mrem, Mremfinal, _to_spawn = split_m(M, tospawn, W) # M is guaranteed to be > W because of `W ≥ M` condition for `jmultsplitn!`...
-    atomicsync = allocref(StaticInt{2}()*num_cores()*cache_linesize())
-    p = reinterpret(Ptr{UInt}, Base.unsafe_convert(Ptr{UInt8}, atomicsync))
+    atomicsync = allocref((StaticInt{1}()+num_cores())*cache_linesize())
+    p = align(reinterpret(Ptr{UInt32}, Base.unsafe_convert(Ptr{UInt8}, atomicsync)))
     GC.@preserve atomicsync begin
-        for i ∈ CloseOpen(2_to_spawn)
-            _atomic_store!(p + i*cache_linesize(), zero(UInt))
+        for i ∈ CloseOpen(_to_spawn)
+            _atomic_store!(reinterpret(Ptr{UInt64}, p) + i*cache_linesize(), 0x0000000000000000)
         end
         Mblock_Mrem, Mblock_ = promote(Mbsize + W, Mbsize)
         u_to_spawn = _to_spawn % UInt
@@ -414,87 +414,81 @@ function matmul_pack_A_and_B!(
 end
 
 function sync_mul!(
-    C::AbstractStridedPointer{T}, A::AbstractStridedPointer, B::AbstractStridedPointer, α, β, M, K, N, atomicp::Ptr{UInt}, bc::Ptr, id::UInt, total_ids::UInt,
+    C::AbstractStridedPointer{T}, A::AbstractStridedPointer, B::AbstractStridedPointer, α, β, M, K, N, atomicp::Ptr{UInt32}, bc::Ptr, id::UInt, total_ids::UInt,
     ::StaticFloat64{W₁}, ::StaticFloat64{W₂}, ::StaticFloat64{R₁}, ::StaticFloat64{R₂}
 ) where {T, W₁, W₂, R₁, R₂}
 
-    (Mblock, Mblock_Mrem, Mremfinal, Mrem, Miter), (Kblock, Kblock_Krem, Krem, Kiter), (Nblock, Nblock_Nrem, Nrem, Niter) =
-        solve_block_sizes(Val(T), M, K, N, StaticFloat64{W₁}(), StaticFloat64{W₂}(), StaticFloat64{R₁}(), StaticFloat64{R₂}(), One())
-
-    # atomics = atomicp + 8sizeof(UInt)
-    sync_iters = zero(UInt)
-    myp = atomicp +   id *cache_linesize()
-    atomicp -= cache_linesize()
-    atomics = atomicp + total_ids*cache_linesize()
-    mys = myp + total_ids*(cache_linesize() % UInt)
-    Npackb_r_div, Npackb_r_rem = divrem_fast(Nblock_Nrem, total_ids)
-    Npackb_r_block_rem, Npackb_r_block_ = promote(Npackb_r_div + One(), Npackb_r_div)
-
-    Npackb___div, Npackb___rem = divrem_fast(Nblock, total_ids)
-    Npackb___block_rem, Npackb___block_ = promote(Npackb___div + One(), Npackb___div)
-
-    pack_r_offset = Npackb_r_div * id + min(id, Npackb_r_rem)
-    pack___offset = Npackb___div * id + min(id, Npackb___rem)
-
-    pack_r_len = ifelse(id < Npackb_r_rem, Npackb_r_block_rem, Npackb_r_block_)
-    pack___len = ifelse(id < Npackb___rem, Npackb___block_rem, Npackb___block_)
-
-    for n in CloseOpen(Niter)
-        # Krem
-        # pack kc x nc block of B
-        nfull = n < Nrem
-        nsize = ifelse(nfull, Nblock_Nrem, Nblock)
-        pack_offset = ifelse(nfull, pack_r_offset, pack___offset)
-        pack_len = ifelse(nfull, pack_r_len, pack___len)
-        let A = A, B = B
-            for k ∈ CloseOpen(Kiter)
-                ksize = ifelse(k < Krem, Kblock_Krem, Kblock)
-                _B = default_zerobased_stridedpointer(bc, (One(), ksize))
-                unsafe_copyto_turbo!(gesp(_B, (Zero(), pack_offset)), gesp(B, (Zero(), pack_offset)), ksize, pack_len)
-                # synchronize before starting the multiplication, to ensure `B` is packed
-                _mv = _atomic_add!(myp, one(UInt))
-                sync_iters += one(UInt)
-                let atomp = atomicp
-                    for _ ∈ CloseOpen(total_ids)
-                        atomp += cache_linesize()
-                        atomp == myp && continue
-                        while _atomic_load(atomp) != sync_iters
-                            pause()
-                        end
-                    end
-                end
-                # multiply
-                let A = A, B = _B, C = C
-                    for m in CloseOpen(Miter)
-                        msize = ifelse((m+1) == Miter, Mremfinal, ifelse(m < Mrem, Mblock_Mrem, Mblock))
-                        if k == 0
-                            packaloopmul!(C, A, B, α,     β, msize, ksize, nsize)
-                        else
-                            packaloopmul!(C, A, B, α, One(), msize, ksize, nsize)
-                        end
-                        A = gesp(A, (msize, Zero()))
-                        C = gesp(C, (msize, Zero()))
-                    end
-                end
-                A = gesp(A, (Zero(), ksize))
-                B = gesp(B, (ksize, Zero()))
-                # synchronize on completion so we wait until every thread is done with `Bpacked` before beginning to overwrite it
-                _mv = _atomic_add!(mys, one(UInt))
-                let atoms = atomics
-                    for _ ∈ CloseOpen(total_ids)
-                        atoms += cache_linesize()
-                        atoms == mys && continue
-                        while _atomic_load(atoms) != sync_iters
-                            pause()
-                        end
-                    end
-                end
+  (Mblock, Mblock_Mrem, Mremfinal, Mrem, Miter), (Kblock, Kblock_Krem, Krem, Kiter), (Nblock, Nblock_Nrem, Nrem, Niter) =
+    solve_block_sizes(Val(T), M, K, N, StaticFloat64{W₁}(), StaticFloat64{W₂}(), StaticFloat64{R₁}(), StaticFloat64{R₂}(), One())
+
+  sync_iters = 0x00000000
+  myp = atomicp + id *cache_linesize()
+  Npackb_r_div, Npackb_r_rem = divrem_fast(Nblock_Nrem, total_ids)
+  Npackb_r_block_rem, Npackb_r_block_ = promote(Npackb_r_div + One(), Npackb_r_div)
+
+  Npackb___div, Npackb___rem = divrem_fast(Nblock, total_ids)
+  Npackb___block_rem, Npackb___block_ = promote(Npackb___div + One(), Npackb___div)
+
+  pack_r_offset = Npackb_r_div * id + min(id, Npackb_r_rem)
+  pack___offset = Npackb___div * id + min(id, Npackb___rem)
+
+  pack_r_len = ifelse(id < Npackb_r_rem, Npackb_r_block_rem, Npackb_r_block_)
+  pack___len = ifelse(id < Npackb___rem, Npackb___block_rem, Npackb___block_)
+
+  for n in CloseOpen(Niter)
+    # Krem
+    # pack kc x nc block of B
+    nfull = n < Nrem
+    nsize = ifelse(nfull, Nblock_Nrem, Nblock)
+    pack_offset = ifelse(nfull, pack_r_offset, pack___offset)
+    pack_len = ifelse(nfull, pack_r_len, pack___len)
+    let A = A, B = B
+      for k ∈ CloseOpen(Kiter)
+        ksize = ifelse(k < Krem, Kblock_Krem, Kblock)
+        _B = default_zerobased_stridedpointer(bc, (One(), ksize))
+        unsafe_copyto_turbo!(gesp(_B, (Zero(), pack_offset)), gesp(B, (Zero(), pack_offset)), ksize, pack_len)
+        # synchronize before starting the multiplication, to ensure `B` is packed
+        _mv = _atomic_add!(myp, 0x00000001)
+        sync_iters += 0x00000001
+        let atomp = atomicp
+          for _ ∈ CloseOpen(total_ids)
+            while _atomic_load(atomp) ≠ sync_iters
+              pause()
             end
+            atomp += cache_linesize()
+          end
         end
-        B = gesp(B, (Zero(), nsize))
-        C = gesp(C, (Zero(), nsize))
+        # multiply
+        let A = A, B = _B, C = C
+          for m in CloseOpen(Miter)
+            msize = ifelse((m+1) == Miter, Mremfinal, ifelse(m < Mrem, Mblock_Mrem, Mblock))
+            if k == 0
+              packaloopmul!(C, A, B, α,     β, msize, ksize, nsize)
+            else
+              packaloopmul!(C, A, B, α, One(), msize, ksize, nsize)
+            end
+            A = gesp(A, (msize, Zero()))
+            C = gesp(C, (msize, Zero()))
+          end
+        end
+        _mv = _atomic_add!(myp + 4, 0x00000001)
+        A = gesp(A, (Zero(), ksize))
+        B = gesp(B, (ksize, Zero()))
+        # synchronize on completion so we wait until every thread is done with `Bpacked` before beginning to overwrite it
+        let atomp = atomicp
+          for _ ∈ CloseOpen(total_ids)
+            while _atomic_load(atomp+4) ≠ sync_iters
+              pause()
+            end
+            atomp += cache_linesize()
+          end
+        end
+      end
     end
-    nothing
+    B = gesp(B, (Zero(), nsize))
+    C = gesp(C, (Zero(), nsize))
+  end
+  nothing
 end
 
 function _matmul!(y::AbstractVector{T}, A::AbstractMatrix, x::AbstractVector, α, β, MKN, contig_axis) where {T<:Real}