KhronosGroup · MrSidims · Oct 20, 2022 · Mar 1, 2022
diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc
@@ -46,3 +46,4 @@ EXT(SPV_INTEL_debug_module)
 EXT(SPV_INTEL_runtime_aligned)
 EXT(SPV_INTEL_arithmetic_fence)
 EXT(SPV_INTEL_bfloat16_conversion)
+EXT(SPV_INTEL_split_barrier)
diff --git a/lib/SPIRV/OCLToSPIRV.cpp b/lib/SPIRV/OCLToSPIRV.cpp
@@ -262,8 +262,11 @@ class OCLToSPIRVBase : public InstVisitor<OCLToSPIRVBase> {
   void visitSubgroupAVCBuiltinCall(CallInst *CI, StringRef DemangledName);
   void visitSubgroupAVCWrapperBuiltinCall(CallInst *CI, Op WrappedOC,
                                           StringRef DemangledName);
+
   void visitSubgroupAVCBuiltinCallWithSampler(CallInst *CI,
                                               StringRef DemangledName);
+  /// For cl_intel_split_work_group_barrier built-ins:
+  void visitCallSplitBarrierINTEL(CallInst *CI, StringRef DemangledName);
 
   void visitCallLdexp(CallInst *CI, StringRef MangledName,
                       StringRef DemangledName);
@@ -584,6 +587,10 @@ void OCLToSPIRVBase::visitCallInst(CallInst &CI) {
     visitSubgroupImageMediaBlockINTEL(&CI, DemangledName);
     return;
   }
+  if (DemangledName.find(kOCLBuiltinName::SplitBarrierINTELPrefix) == 0) {
+    visitCallSplitBarrierINTEL(&CI, DemangledName);
+    return;
+  }
   // Handle 'cl_intel_device_side_avc_motion_estimation' extension built-ins
   if (DemangledName.find(kOCLSubgroupsAVCIntel::Prefix) == 0 ||
       // Workaround for a bug in the extension specification
@@ -2048,6 +2055,37 @@ void OCLToSPIRVBase::visitSubgroupAVCBuiltinCallWithSampler(
       &Attrs);
 }
 
+void OCLToSPIRVBase::visitCallSplitBarrierINTEL(CallInst *CI,
+                                                StringRef DemangledName) {
+  auto Lit = getBarrierLiterals(CI);
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  Op OpCode =
+      StringSwitch<Op>(DemangledName)
+          .Case("intel_work_group_barrier_arrive", OpControlBarrierArriveINTEL)
+          .Case("intel_work_group_barrier_wait", OpControlBarrierWaitINTEL)
+          .Default(OpNop);
+
+  mutateCallInstSPIRV(
+      M, CI,
+      [=](CallInst *, std::vector<Value *> &Args) {
+        Args.resize(3);
+        // Execution scope
+        Args[0] = addInt32(map<Scope>(std::get<2>(Lit)));
+        // Memory scope
+        Args[1] = addInt32(map<Scope>(std::get<1>(Lit)));
+        // Memory semantics
+        // OpControlBarrierArriveINTEL -> Release,
+        // OpControlBarrierWaitINTEL -> Acquire
+        unsigned MemFenceFlag = std::get<0>(Lit);
+        OCLMemOrderKind MemOrder = OpCode == OpControlBarrierArriveINTEL
+                                       ? OCLMO_release
+                                       : OCLMO_acquire;
+        Args[2] = addInt32(mapOCLMemSemanticToSPIRV(MemFenceFlag, MemOrder));
+        return getSPIRVFuncName(OpCode);
+      },
+      &Attrs);
+}
+
 void OCLToSPIRVBase::visitCallLdexp(CallInst *CI, StringRef MangledName,
                                     StringRef DemangledName) {
   auto Args = getArguments(CI);

diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp
@@ -421,6 +421,9 @@ template <> void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
   _SPIRV_OP(bitfield_extract_signed, BitFieldSExtract)
   _SPIRV_OP(bitfield_extract_unsigned, BitFieldUExtract)
   _SPIRV_OP(bit_reverse, BitReverse)
+  // cl_khr_split_work_group_barrier
+  _SPIRV_OP(intel_work_group_barrier_arrive, ControlBarrierArriveINTEL)
+  _SPIRV_OP(intel_work_group_barrier_wait, ControlBarrierWaitINTEL)
 #undef _SPIRV_OP
 }
 
@@ -1026,7 +1029,9 @@ class OCLBuiltinFuncMangleInfo : public SPIRV::BuiltinFuncMangleInfo {
     } else if (NameRef.contains("barrier")) {
       addUnsignedArg(0);
       if (NameRef.equals("work_group_barrier") ||
-          NameRef.equals("sub_group_barrier"))
+          NameRef.equals("sub_group_barrier") ||
+          NameRef.equals("intel_work_group_barrier_arrive") ||
+          NameRef.equals("intel_work_group_barrier_wait"))
         setEnumArg(1, SPIR::PRIMITIVE_MEMORY_SCOPE);
     } else if (NameRef.startswith("atomic_work_item_fence")) {
       addUnsignedArg(0);

diff --git a/lib/SPIRV/OCLUtil.h b/lib/SPIRV/OCLUtil.h
@@ -307,6 +307,7 @@ const static char SubgroupBlockWriteINTELPrefix[] =
     "intel_sub_group_block_write";
 const static char SubgroupImageMediaBlockINTELPrefix[] =
     "intel_sub_group_media_block";
+const static char SplitBarrierINTELPrefix[] = "intel_work_group_barrier_";
 const static char LDEXP[] = "ldexp";
 #define _SPIRV_OP(x)                                                           \
   const static char ConvertBFloat16##x##AsUShort##x[] =                        \

diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp
@@ -2950,7 +2950,9 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName,
     if (isFuncNoUnwind())
       Func->addFnAttr(Attribute::NoUnwind);
     auto OC = BI->getOpCode();
-    if (isGroupOpCode(OC) || isIntelSubgroupOpCode(OC))
+    if (isGroupOpCode(OC) || isIntelSubgroupOpCode(OC) ||
+        isSplitBarrierINTELOpCode(OC))
+
       Func->addFnAttr(Attribute::Convergent);
   }
   auto Call =

diff --git a/lib/SPIRV/SPIRVToOCL.cpp b/lib/SPIRV/SPIRVToOCL.cpp
@@ -107,6 +107,10 @@ void SPIRVToOCLBase::visitCallInst(CallInst &CI) {
   if (OC == OpControlBarrier) {
     visitCallSPIRVControlBarrier(&CI);
   }
+  if (isSplitBarrierINTELOpCode(OC)) {
+    visitCallSPIRVSplitBarrierINTEL(&CI, OC);
+    return;
+  }
   if (isAtomicOpCode(OC)) {
     visitCallSPIRVAtomicBuiltin(&CI, OC);
     return;

diff --git a/lib/SPIRV/SPIRVToOCL.h b/lib/SPIRV/SPIRVToOCL.h
@@ -222,6 +222,11 @@ class SPIRVToOCLBase : public InstVisitor<SPIRVToOCLBase> {
   /// - OCL1.2: barrier
   virtual void visitCallSPIRVControlBarrier(CallInst *CI) = 0;
 
+  /// Transform split __spirv_ControlBarrier barrier to:
+  /// - OCL2.0: overload with a memory_scope argument
+  /// - OCL1.2: overload with no memory_scope argument
+  virtual void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) = 0;
+
   /// Transform __spirv_EnqueueKernel to __enqueue_kernel
   virtual void visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) = 0;
 
@@ -290,6 +295,10 @@ class SPIRVToOCL12Base : public SPIRVToOCLBase {
   ///       barrier(flag(sema))
   void visitCallSPIRVControlBarrier(CallInst *CI) override;
 
+  /// Transform split __spirv_ControlBarrier barrier to overloads without a
+  /// memory_scope argument.
+  void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override;
+
   /// Transform __spirv_OpAtomic functions. It firstly conduct generic
   /// mutations for all builtins and then mutate some of them seperately
   Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) override;
@@ -379,6 +388,10 @@ class SPIRVToOCL20Base : public SPIRVToOCLBase {
   ///         sub_group_barrier(flag(sema), map(memScope))
   void visitCallSPIRVControlBarrier(CallInst *CI) override;
 
+  /// Transform split __spirv_ControlBarrier barrier to overloads with a
+  /// memory_scope argument.
+  void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override;
+
   /// Transform __spirv_Atomic* to atomic_*.
   ///   __spirv_Atomic*(atomic_op, scope, sema, ops, ...) =>
   ///      atomic_*(generic atomic_op, ops, ..., order(sema), map(scope))

diff --git a/lib/SPIRV/SPIRVToOCL12.cpp b/lib/SPIRV/SPIRVToOCL12.cpp
@@ -102,6 +102,19 @@ void SPIRVToOCL12Base::visitCallSPIRVControlBarrier(CallInst *CI) {
       &Attrs);
 }
 
+void SPIRVToOCL12Base::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) {
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  mutateCallInstOCL(
+      M, CI,
+      [=](CallInst *, std::vector<Value *> &Args) {
+        Value *MemFenceFlags =
+            SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI);
+        Args.assign(1, MemFenceFlags);
+        return OCLSPIRVBuiltinMap::rmap(OC);
+      },
+      &Attrs);
+}
+
 Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) {
   AttributeList Attrs = CI->getCalledFunction()->getAttributes();
   return mutateCallInstOCL(

diff --git a/lib/SPIRV/SPIRVToOCL20.cpp b/lib/SPIRV/SPIRVToOCL20.cpp
@@ -121,6 +121,28 @@ void SPIRVToOCL20Base::visitCallSPIRVControlBarrier(CallInst *CI) {
       &Attrs);
 }
 
+void SPIRVToOCL20Base::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) {
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  mutateCallInstOCL(
+      M, CI,
+      [=](CallInst *, std::vector<Value *> &Args) {
+        auto GetArg = [=](unsigned I) {
+          return cast<ConstantInt>(Args[I])->getZExtValue();
+        };
+        Value *MemScope =
+            getInt32(M, rmap<OCLScopeKind>(static_cast<Scope>(GetArg(1))));
+        Value *MemFenceFlags =
+            SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI);
+
+        Args.resize(2);
+        Args[0] = MemFenceFlags;
+        Args[1] = MemScope;
+
+        return OCLSPIRVBuiltinMap::rmap(OC);
+      },
+      &Attrs);
+}
+
 std::string SPIRVToOCL20Base::mapFPAtomicName(Op OC) {
   assert(isFPAtomicOpCode(OC) && "Not intended to handle other opcodes than "
                                  "AtomicF{Add/Min/Max}EXT!");

diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h
@@ -3269,6 +3269,25 @@ class SPIRVBfloat16ConversionINTELInstBase : public SPIRVUnaryInst<OC> {
 _SPIRV_OP(ConvertFToBF16INTEL)
 _SPIRV_OP(ConvertBF16ToFINTEL)
 #undef _SPIRV_OP
+
+class SPIRVSplitBarrierINTELBase : public SPIRVInstTemplateBase {
+protected:
+  SPIRVCapVec getRequiredCapability() const override {
+    return getVec(CapabilitySplitBarrierINTEL);
+  }
+
+  llvm::Optional<ExtensionID> getRequiredExtension() const override {
+    return ExtensionID::SPV_INTEL_split_barrier;
+  }
+};
+
+#define _SPIRV_OP(x, ...)                                                      \
+  typedef SPIRVInstTemplate<SPIRVSplitBarrierINTELBase, Op##x, __VA_ARGS__>    \
+      SPIRV##x;
+_SPIRV_OP(ControlBarrierArriveINTEL, false, 4)
+_SPIRV_OP(ControlBarrierWaitINTEL, false, 4)
+#undef _SPIRV_OP
+
 } // namespace SPIRV
 
 #endif // SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H
diff --git a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
@@ -572,6 +572,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(internal::CapabilityOptNoneINTEL, "OptNoneINTEL");
   add(internal::CapabilityMemoryAccessAliasingINTEL,
       "MemoryAccessAliasingINTEL");
+  add(CapabilitySplitBarrierINTEL, "SplitBarrierINTEL");
   add(internal::CapabilityFPGAInvocationPipeliningAttributesINTEL,
       "FPGAInvocationPipeliningAttributesINTEL");
   add(internal::CapabilityTokenTypeINTEL, "TokenTypeINTEL");

diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCode.h b/lib/SPIRV/libSPIRV/SPIRVOpCode.h
@@ -243,6 +243,11 @@ inline bool isEventOpCode(Op OpCode) {
   return OpRetainEvent <= OpCode && OpCode <= OpCaptureEventProfilingInfo;
 }
 
+inline bool isSplitBarrierINTELOpCode(Op OpCode) {
+  return OpCode == OpControlBarrierArriveINTEL ||
+         OpCode == OpControlBarrierWaitINTEL;
+}
+
 } // namespace SPIRV
 
 #endif // SPIRV_LIBSPIRV_SPIRVOPCODE_H
diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h b/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h
@@ -543,3 +543,5 @@ _SPIRV_OP(TypeBufferSurfaceINTEL, 6086)
 _SPIRV_OP(TypeStructContinuedINTEL, 6090)
 _SPIRV_OP(ConstantCompositeContinuedINTEL, 6091)
 _SPIRV_OP(SpecConstantCompositeContinuedINTEL, 6092)
+_SPIRV_OP(ControlBarrierArriveINTEL, 6142)
+_SPIRV_OP(ControlBarrierWaitINTEL, 6143)
diff --git a/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_12.ll b/test/transcoding/SPV_INTEL_split_barrier/split_work_group_barrier_12.ll
@@ -0,0 +1,99 @@
+;; kernel void test(global uint* dst)
+;; {
+;;     intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE);
+;;     intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE);
+;;     intel_work_group_barrier_arrive(CLK_GLOBAL_MEM_FENCE);
+;;     intel_work_group_barrier_wait(CLK_GLOBAL_MEM_FENCE);
+;;
+;;     intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
+;;     intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
+;;}
+
+; Test for SPV_INTEL_split_barrier (OpenCL C LLVM IR)
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+SPV_INTEL_split_barrier
+; RUN: llvm-spirv %t.spv -o %t.spt --to-text
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.spv -o %t.rev.bc -r --spirv-target-env=CL1.2
+; RUN: llvm-dis %t.rev.bc -o %t.rev.ll
+; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM
+
+; RUN: not llvm-spirv %t.bc 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; CHECK-ERROR: RequiresExtension: Feature requires the following SPIR-V extension:
+; CHECK-ERROR-NEXT: SPV_INTEL_split_barrier
+
+; ModuleID = 'split_barrier.cl'
+source_filename = "split_barrier.cl"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+; CHECK-SPIRV: Capability SplitBarrierINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_split_barrier"
+; CHECK-SPIRV: TypeInt [[UINT:[0-9]+]] 32 0
+;
+; Scopes:
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_WORK_GROUP:[0-9]+]] 2
+;
+; Memory Semantics:
+; 0x2 Acquire + 0x100 WorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQUIRE_LOCAL:[0-9]+]] 258
+; 0x4 Release + 0x100 WorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[RELEASE_LOCAL:[0-9]+]] 260
+; 0x2 Acquire + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQUIRE_GLOBAL:[0-9]+]] 514
+; 0x4 Release + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[RELEASE_GLOBAL:[0-9]+]] 516
+; 0x2 Acquire + 0x100 WorkgroupMemory + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQUIRE_LOCAL_GLOBAL:[0-9]+]] 770
+; 0x4 Release + 0x100 WorkgroupMemory + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[RELEASE_LOCAL_GLOBAL:[0-9]+]] 772
+;
+; CHECK-SPIRV: ControlBarrierArriveINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[RELEASE_LOCAL]]
+; CHECK-SPIRV: ControlBarrierWaitINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[ACQUIRE_LOCAL]]
+; CHECK-SPIRV: ControlBarrierArriveINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[RELEASE_GLOBAL]]
+; CHECK-SPIRV: ControlBarrierWaitINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[ACQUIRE_GLOBAL]]
+;
+; CHECK-SPIRV: ControlBarrierArriveINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[RELEASE_LOCAL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrierWaitINTEL [[SCOPE_WORK_GROUP]] [[SCOPE_WORK_GROUP]] [[ACQUIRE_LOCAL_GLOBAL]]
+
+; CHECK-LLVM-LABEL: define spir_kernel void @test
+; Function Attrs: convergent norecurse nounwind
+define dso_local spir_kernel void @test(i32 addrspace(1)* nocapture noundef readnone align 4 %0) local_unnamed_addr #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
+  tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 1) #2
+    ; CHECK-LLVM: call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 1)
+  tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 1) #2
+    ; CHECK-LLVM: call spir_func void @_Z29intel_work_group_barrier_waitj(i32 1)
+  tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 2) #2
+    ; CHECK-LLVM: call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 2)
+  tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 2) #2
+    ; CHECK-LLVM: call spir_func void @_Z29intel_work_group_barrier_waitj(i32 2)
+  tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 3) #2
+    ; CHECK-LLVM: call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 3)
+  tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 3) #2
+    ; CHECK-LLVM: call spir_func void @_Z29intel_work_group_barrier_waitj(i32 3)
+  ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nounwind }
+
+!llvm.module.flags = !{!0, !1}
+!opencl.ocl.version = !{!2}
+!opencl.spir.version = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"frame-pointer", i32 2}
+!2 = !{i32 1, i32 2}
+!3 = !{!"clang version 15.0.0 (https://github.com/llvm/llvm-project 861386dbd6ff0d91636b7c674c2abb2eccd9d3f2)"}
+!4 = !{i32 1}
+!5 = !{!"none"}
+!6 = !{!"uint*"}
+!7 = !{!""}