[SYCL] Support for load/store cache controls

llvm/lib/SYCLLowerIR/CompileTimeProperties.def

@@ -54,3 +54,7 @@ SYCL_COMPILE_TIME_PROPERTY("sycl-stride-size", 5883, DecorValueTy::uint32)
 SYCL_COMPILE_TIME_PROPERTY("sycl-word-size", 5884, DecorValueTy::uint32)
 SYCL_COMPILE_TIME_PROPERTY("sycl-bi-directional-ports-true", 5885,
                            DecorValueTy::none)
+
+// The corresponding SPIR-V OpCodes for cache control properties
+SYCL_COMPILE_TIME_PROPERTY("sycl-cache-read-hint", 6442, DecorValueTy::uint32)
+SYCL_COMPILE_TIME_PROPERTY("sycl-cache-write-hint", 6443, DecorValueTy::uint32)

llvm/lib/SYCLLowerIR/CompileTimePropertiesPass.cpp

@@ -42,6 +42,9 @@ constexpr uint32_t SPIRV_HOST_ACCESS_DEFAULT_VALUE = 2; // Read/Write
 constexpr uint32_t SPIRV_INITIATION_INTERVAL_DECOR = 5917;
 constexpr uint32_t SPIRV_PIPELINE_ENABLE_DECOR = 5919;
 
+constexpr uint32_t SPIRV_CACHE_CONTROL_READ_DECOR = 6442;
+constexpr uint32_t SPIRV_CACHE_CONTROL_WRITE_DECOR = 6443;
+
 enum class DecorValueTy {
   uint32,
   boolean,
@@ -97,6 +100,95 @@ MDNode *buildSpirvDecorMetadata(LLVMContext &Ctx, uint32_t OpCode,
   return MDNode::get(Ctx, MD);
 }
 
+/// Builds a metadata node for a SPIR-V decoration for cache controls
+/// where decoration code and value are both uint32_t integers.
+/// The value encodes a cache level and a cache control type.
+///
+/// @param Ctx        [in] the LLVM Context.
+/// @param Name       [in] the SPIR-V property string name.
+/// @param OpCode     [in] the SPIR-V opcode.
+/// @param CacheMode  [in] whether read or write.
+/// @param CacheLevel [in] the cache level.
+///
+/// @returns a pointer to the metadata node created for the required decoration
+/// and its values.
+MDNode *buildSpirvDecorCacheProp(LLVMContext &Ctx, StringRef Name,
+                                 uint32_t OpCode, uint32_t CacheMode,
+                                 uint32_t CacheLevel) {
+  // SPIR-V encodings of read control
+  enum cache_control_read_type {
+    read_uncached = 0,
+    read_cached = 1,
+    read_streaming = 2,
+    read_invalidate = 3,
+    read_const_cached = 4
+  };
+  // SPIR-V encodings of write control
+  enum cache_control_write_type {
+    write_uncached = 0,
+    write_through = 1,
+    write_back = 2,
+    write_streaming = 3
+  };
+  // SYCL encodings of read/write control
+  enum cache_mode {
+    uncached,
+    cached,
+    streaming,
+    invalidate,
+    const_cached,
+    through,
+    back
+  };
+
+  // Map SYCL encoding to SPIR-V
+  uint32_t CacheProp;
+  if (Name == "sycl-cache-read-hint") {
+    switch (CacheMode) {
+    case uncached:
+      CacheProp = read_uncached;
+      break;
+    case cached:
+      CacheProp = read_cached;
+      break;
+    case streaming:
+      CacheProp = read_streaming;
+      break;
+    case invalidate:
+      CacheProp = read_invalidate;
+      break;
+    case const_cached:
+      CacheProp = read_const_cached;
+      break;
+    }
+  } else {
+    switch (CacheMode) {
+    case uncached:
+      CacheProp = write_uncached;
+      break;
+    case through:
+      CacheProp = write_through;
+      break;
+    case back:
+      CacheProp = write_back;
+      break;
+    case streaming:
+      CacheProp = write_streaming;
+      break;
+    }
+  }
+
+  auto *Ty = Type::getInt32Ty(Ctx);
+  SmallVector<Metadata *, 3> MD;
+  MD.push_back(ConstantAsMetadata::get(
+      Constant::getIntegerValue(Ty, APInt(32, OpCode))));
+  MD.push_back(ConstantAsMetadata::get(
+      Constant::getIntegerValue(Ty, APInt(32, CacheLevel))));
+  MD.push_back(ConstantAsMetadata::get(
+      Constant::getIntegerValue(Ty, APInt(32, CacheProp))));
+  return MDNode::get(Ctx, MD);
+}
+
 /// Builds a metadata node for a SPIR-V decoration (decoration code
 /// is \c uint32_t integer and value is a string).
 ///
@@ -625,9 +717,12 @@ bool CompileTimePropertiesPass::transformSYCLPropertiesAnnotation(
   // check alignment annotation and apply it to load/store
   parseAlignmentAndApply(M, IntrInst);
 
-  // Read the annotation values and create the new annotation string.
+  // Read the annotation values and create new annotation strings.
   std::string NewAnnotString = "";
   auto Properties = parseSYCLPropertiesString(M, IntrInst);
+  SmallVector<Metadata *, 8> MDOpsCacheProp;
+  bool CacheProp = false;
+  bool FPGAProp = false;
   for (const auto &[PropName, PropVal] : Properties) {
     // sycl-alignment is converted to align on
     // previous parseAlignmentAndApply(), dropping here
@@ -639,51 +734,108 @@ bool CompileTimePropertiesPass::transformSYCLPropertiesAnnotation(
       continue;
     uint32_t DecorCode = DecorIt->second.Code;
 
-    // Expected format is '{X}' or '{X:Y}' where X is decoration ID and
-    // Y is the value if present. It encloses Y in " to ensure that
-    // string values are handled correctly. Note that " around values are
-    // always valid, even if the decoration parameters are not strings.
-    NewAnnotString += "{" + std::to_string(DecorCode);
-    if (PropVal)
-      NewAnnotString += ":\"" + PropVal->str() + "\"";
-    NewAnnotString += "}";
+    // Handle cache control properties
+    if ((*PropName).starts_with("sycl-cache-")) {
+      CacheProp = true;
+      auto DecorValue = PropVal;
+      uint32_t AttrVal;
+      DecorValue->getAsInteger(0, AttrVal);
+      // Format is:
+      // !Annot = !{!CC1, !CC2, ...}
+      // !CC1 = !{i32 Load/Store, i32 Level, i32 Control}
+      // !CC2 = !{i32 Load/Store, i32 Level, i32 Control}
+      // ...
+      LLVMContext &Ctx = M.getContext();
+      uint32_t CacheMode = 0;
+      while (AttrVal) {
+        // The attribute value encodes cache control and levels.
+        // Low-order to high-order nibbles represent the enumerated cache modes.
+        // In each nibble cache levels are encodes as L1->bit0, L2->bit1,
+        // L3->bit2 and L4->bit3. The SPIR-V encoding uses numbers 0..3.
+        uint32_t CacheLevel = 0;
+        uint32_t LevelMask = AttrVal & 0xf;
+        while (LevelMask) {
+          if (LevelMask & 1)
+            MDOpsCacheProp.push_back(buildSpirvDecorCacheProp(
+                Ctx, *PropName, DecorCode, CacheMode, CacheLevel));
+          ++CacheLevel;
+          LevelMask >>= 1;
+        }
+        ++CacheMode;
+        AttrVal >>= 4;
+      }
+    } else {
+      FPGAProp = true;
+      // Expected format is '{X}' or '{X:Y}' where X is decoration ID and
+      // Y is the value if present. It encloses Y in " to ensure that
+      // string values are handled correctly. Note that " around values are
+      // always valid, even if the decoration parameters are not strings.
+      NewAnnotString += "{" + std::to_string(DecorCode);
+      if (PropVal)
+        NewAnnotString += ":\"" + PropVal->str() + "\"";
+      NewAnnotString += "}";
+    }
   }
 
-  // If the new annotation string is empty there is no reason to keep it, so
-  // replace it with the first operand and mark it for removal.
-  if (NewAnnotString.empty()) {
+  // If there are no other annotations (except "alignment") then there is no
+  // reason to keep the original intrinsic, so replace it with the first operand
+  // and mark it for removal.
+  if (!CacheProp && !FPGAProp) {
     IntrInst->replaceAllUsesWith(IntrInst->getOperand(0));
     RemovableAnnotations.push_back(IntrInst);
     return true;
   }
 
-  // Either reuse a previously generated one or create a new global variable
-  // with the new annotation string.
-  GlobalVariable *NewAnnotStringGV = nullptr;
-  auto ExistingNewAnnotStringIt = ReusableAnnotStrings.find(NewAnnotString);
-  if (ExistingNewAnnotStringIt != ReusableAnnotStrings.end()) {
-    NewAnnotStringGV = ExistingNewAnnotStringIt->second;
-  } else {
-    Constant *NewAnnotStringData =
-        ConstantDataArray::getString(M.getContext(), NewAnnotString);
-    NewAnnotStringGV = new GlobalVariable(
-        M, NewAnnotStringData->getType(), true, GlobalValue::PrivateLinkage,
-        NewAnnotStringData, ".str", nullptr, llvm::GlobalValue::NotThreadLocal,
-        IntrAnnotStringArg->getType()->getPointerAddressSpace());
-    NewAnnotStringGV->setSection(AnnotStrArgGV->getSection());
-    NewAnnotStringGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
-    ReusableAnnotStrings.insert({NewAnnotString, NewAnnotStringGV});
+  if (FPGAProp) {
+    // Either reuse a previously generated one or create a new global variable
+    // with the new annotation string.
+    GlobalVariable *NewAnnotStringGV = nullptr;
+    auto ExistingNewAnnotStringIt = ReusableAnnotStrings.find(NewAnnotString);
+    if (ExistingNewAnnotStringIt != ReusableAnnotStrings.end()) {
+      NewAnnotStringGV = ExistingNewAnnotStringIt->second;
+    } else {
+      Constant *NewAnnotStringData =
+          ConstantDataArray::getString(M.getContext(), NewAnnotString);
+      NewAnnotStringGV = new GlobalVariable(
+          M, NewAnnotStringData->getType(), true, GlobalValue::PrivateLinkage,
+          NewAnnotStringData, ".str", nullptr,
+          llvm::GlobalValue::NotThreadLocal,
+          IntrAnnotStringArg->getType()->getPointerAddressSpace());
+      NewAnnotStringGV->setSection(AnnotStrArgGV->getSection());
+      NewAnnotStringGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+      ReusableAnnotStrings.insert({NewAnnotString, NewAnnotStringGV});
+    }
+
+    // Replace the annotation string with a bitcast of the new global variable.
+    IntrInst->setArgOperand(
+        1, ConstantExpr::getBitCast(NewAnnotStringGV,
+                                    IntrAnnotStringArg->getType()));
+
+    // The values are now in the annotation string, so we can remove the
+    // original annotation value.
+    PointerType *Arg4PtrTy =
+        cast<PointerType>(IntrInst->getArgOperand(4)->getType());
+    IntrInst->setArgOperand(4, ConstantPointerNull::get(Arg4PtrTy));
   }
 
-  // Replace the annotation string with a bitcast of the new global variable.
-  IntrInst->setArgOperand(
-      1, ConstantExpr::getBitCast(NewAnnotStringGV,
-                                  IntrAnnotStringArg->getType()));
+  if (CacheProp) {
+    LLVMContext &Ctx = M.getContext();
+    unsigned MDKindID = Ctx.getMDKindID(SPIRV_DECOR_MD_KIND);
+    if (!FPGAProp) {
+      // If there are no annotations other than cache controls we can apply the
+      // controls to the pointer and remove the intrinsic.
+      auto PtrInstr = cast<Instruction>(IntrInst->getArgOperand(0));
+      PtrInstr->setMetadata(MDKindID, MDTuple::get(Ctx, MDOpsCacheProp));
+      // Replace all uses of IntrInst with first operand
+      IntrInst->replaceAllUsesWith(PtrInstr);
+      // Delete the original IntrInst
+      RemovableAnnotations.push_back(IntrInst);
+    } else {
+      // If there were FPGA annotations then we retain the original intrinsic
+      // and apply the cache control properties to its result.
+      IntrInst->setMetadata(MDKindID, MDTuple::get(Ctx, MDOpsCacheProp));
+    }
+  }
 
-  // The values are not in the annotation string, so we can remove the original
-  // annotation value.
-  PointerType *Arg4PtrTy =
-      cast<PointerType>(IntrInst->getArgOperand(4)->getType());
-  IntrInst->setArgOperand(4, ConstantPointerNull::get(Arg4PtrTy));
   return true;
 }