Merge branch 'main' of https://github.com/KhronosGroup/SPIRV-LLVM-Translator into amd-develop

Author: AlexVlx

include/LLVMSPIRVExtensions.inc

@@ -81,6 +81,9 @@ EXT(SPV_INTEL_bfloat16_arithmetic)
 EXT(SPV_INTEL_ternary_bitwise_function)
 EXT(SPV_INTEL_int4)
 EXT(SPV_INTEL_function_variants)
-EXT(SPV_INTEL_shader_atomic_bfloat16)
+EXT(SPV_INTEL_16bit_atomics)
 EXT(SPV_EXT_float8)
 EXT(SPV_INTEL_predicated_io)
+EXT(SPV_INTEL_sigmoid)
+EXT(SPV_INTEL_float4)
+EXT(SPV_INTEL_fp_conversions)

lib/SPIRV/LLVMToSPIRVDbgTran.cpp

@@ -1286,17 +1286,8 @@ SPIRVEntry *LLVMToSPIRVDbgTran::transDbgFunction(const DISubprogram *Func) {
     Ops[FunctionIdIdx] = getDebugInfoNoneId();
     for (const llvm::Function &F : M->functions()) {
       if (Func->describes(&F)) {
-        // Function definition of spir_kernel can have no "spir_kernel" calling
-        // convention because SPIRVRegularizeLLVMBase::addKernelEntryPoint pass
-        // could have turned it to spir_func. The "true" entry point is a
-        // wrapper kernel function, which can be found further in the module.
-        if (FuncDef) {
-          if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
-            IsEntryPointKernel = true;
-            break;
-          }
+        if (FuncDef)
           continue;
-        }
 
         SPIRVValue *SPIRVFunc = SPIRVWriter->getTranslatedValue(&F);
         assert(SPIRVFunc && "All function must be already translated");
@@ -1305,7 +1296,6 @@ SPIRVEntry *LLVMToSPIRVDbgTran::transDbgFunction(const DISubprogram *Func) {
         if (!isNonSemanticDebugInfo())
           break;
 
-        // Most likely unreachable because of Regularise LLVM pass
         if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
           IsEntryPointKernel = true;
           break;

lib/SPIRV/OCLToSPIRV.cpp

@@ -42,15 +42,21 @@
 #include "SPIRVInternal.h"
 #include "libSPIRV/SPIRVDebug.h"
 
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Operator.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/TypedPointerType.h"
 #include "llvm/Support/Debug.h"
 
 #include <algorithm>
+#include <optional>
 #include <regex>
 #include <set>
 
@@ -62,6 +68,88 @@ using namespace SPIRV;
 using namespace OCLUtil;
 
 namespace SPIRV {
+
+static unsigned getAddressSpaceFromType(const Type *Ty) {
+  assert(Ty && "Can't deduce pointer AS");
+  if (auto *TypedPtr = dyn_cast<TypedPointerType>(Ty))
+    return TypedPtr->getAddressSpace();
+  if (auto *Ptr = dyn_cast<PointerType>(Ty))
+    return Ptr->getAddressSpace();
+  llvm_unreachable("Can't deduce pointer AS");
+}
+
+// Performs an address space inference analysis.
+static unsigned getAddressSpaceFromValue(const Value *Ptr) {
+  assert(Ptr && "Can't deduce pointer AS");
+
+  SmallPtrSet<const Value *, 8> Visited;
+  SmallVector<const Value *, 8> Worklist;
+  Worklist.push_back(Ptr);
+  unsigned AS = SPIRAS_Generic;
+
+  while (!Worklist.empty()) {
+    const Value *Current = Worklist.pop_back_val();
+    if (!Visited.insert(Current).second)
+      continue;
+
+    unsigned DeducedAS = getAddressSpaceFromType(Current->getType());
+    if (DeducedAS != SPIRAS_Generic)
+      return DeducedAS;
+    AS = DeducedAS;
+
+    // Find origins of the pointer and add to the worklist.
+    if (auto *Op = dyn_cast<Operator>(Current)) {
+      switch (Op->getOpcode()) {
+      case Instruction::AddrSpaceCast:
+      case Instruction::BitCast:
+      case Instruction::GetElementPtr:
+        Worklist.push_back(Op->getOperand(0));
+        break;
+      case Instruction::Select:
+        Worklist.push_back(Op->getOperand(1));
+        Worklist.push_back(Op->getOperand(2));
+        break;
+      case Instruction::PHI: {
+        auto *Phi = cast<PHINode>(Op);
+        for (Value *Incoming : Phi->incoming_values())
+          Worklist.push_back(Incoming);
+        break;
+      }
+      default:
+        break;
+      }
+    }
+  }
+
+  return AS;
+}
+
+// Sets memory semantic mask of an atomic depending on a pointer argument
+// address space.
+static unsigned
+getAtomicPointerMemorySemanticsMemoryMask(const Value *Ptr,
+                                          const Type *RecordedType) {
+  assert((Ptr && RecordedType) &&
+         "Can't evaluate atomic builtin's memory semantic");
+  unsigned AddrSpace = getAddressSpaceFromType(RecordedType);
+  if (AddrSpace == SPIRAS_Generic)
+    AddrSpace = getAddressSpaceFromValue(Ptr);
+
+  switch (AddrSpace) {
+  case SPIRAS_Global:
+  case SPIRAS_GlobalDevice:
+  case SPIRAS_GlobalHost:
+    return MemorySemanticsCrossWorkgroupMemoryMask;
+  case SPIRAS_Local:
+    return MemorySemanticsWorkgroupMemoryMask;
+  case SPIRAS_Generic:
+    return MemorySemanticsCrossWorkgroupMemoryMask |
+           MemorySemanticsWorkgroupMemoryMask;
+  default:
+    return MemorySemanticsMaskNone;
+  }
+}
+
 static size_t getOCLCpp11AtomicMaxNumOps(StringRef Name) {
   return StringSwitch<size_t>(Name)
       .Cases({"load", "flag_test_and_set", "flag_clear"}, 3)
@@ -704,6 +792,11 @@ void OCLToSPIRVBase::transAtomicBuiltin(CallInst *CI,
   const size_t ScopeIdx = ArgsCount - 1;
   const size_t OrderIdx = ScopeIdx - NumOrder;
 
+  unsigned PtrMemSemantics = MemorySemanticsMaskNone;
+  if (Mutator.arg_size() > 0)
+    PtrMemSemantics = getAtomicPointerMemorySemanticsMemoryMask(
+        Mutator.getArg(0), Mutator.getType(0));
+
   if (NeedsNegate) {
     Mutator.mapArg(1, [=](Value *V) {
       IRBuilder<> IRB(CI);
@@ -714,9 +807,20 @@ void OCLToSPIRVBase::transAtomicBuiltin(CallInst *CI,
     return transOCLMemScopeIntoSPIRVScope(V, OCLMS_device, CI);
   });
   for (size_t I = 0; I < NumOrder; ++I) {
-    Mutator.mapArg(OrderIdx + I, [=](Value *V) {
-      return transOCLMemOrderIntoSPIRVMemorySemantics(V, OCLMO_seq_cst, CI);
-    });
+    Mutator.mapArg(
+        OrderIdx + I, [=](IRBuilder<> &Builder, Value *V) -> Value * {
+          Value *MemSem =
+              transOCLMemOrderIntoSPIRVMemorySemantics(V, OCLMO_seq_cst, CI);
+          if (PtrMemSemantics == MemorySemanticsMaskNone)
+            return MemSem;
+
+          auto *MemSemTy = cast<IntegerType>(MemSem->getType());
+          auto *Mask = ConstantInt::get(MemSemTy, PtrMemSemantics);
+          if (auto *Const = dyn_cast<ConstantInt>(MemSem))
+            return static_cast<Value *>(ConstantInt::get(
+                MemSemTy, Const->getZExtValue() | PtrMemSemantics));
+          return Builder.CreateOr(MemSem, Mask);
+        });
   }
 
   // Order of args in SPIR-V:

lib/SPIRV/SPIRVInternal.h

@@ -1050,6 +1050,7 @@ enum FPEncodingWrap {
   BF16 = FPEncoding::FPEncodingBFloat16KHR,
   E4M3 = FPEncoding::FPEncodingFloat8E4M3EXT,
   E5M2 = FPEncoding::FPEncodingFloat8E5M2EXT,
+  E2M1 = internal::FPEncodingFloat4E2M1INTEL,
 };
 
 // Structure describing non-trivial conversions (FP8 and int4)
@@ -1078,36 +1079,117 @@ typedef SPIRVMap<llvm::StringRef, FPConversionDesc> FPConvertToEncodingMap;
 
 // clang-format off
 template <> inline void FPConvertToEncodingMap::init() {
-   // 8-bit conversions
-   add("ConvertE4M3ToFP16EXT",
-       {FPEncodingWrap::E4M3,       FPEncodingWrap::IEEE754,    OpFConvert});
-   add("ConvertE5M2ToFP16EXT",
-       {FPEncodingWrap::E5M2,       FPEncodingWrap::IEEE754,    OpFConvert});
-   add("ConvertE4M3ToBF16EXT",
-       {FPEncodingWrap::E4M3,       FPEncodingWrap::BF16,       OpFConvert});
-   add("ConvertE5M2ToBF16EXT",
-       {FPEncodingWrap::E5M2,       FPEncodingWrap::BF16,       OpFConvert});
-   add("ConvertFP16ToE4M3EXT",
-       {FPEncodingWrap::IEEE754,    FPEncodingWrap::E4M3,       OpFConvert});
-   add("ConvertFP16ToE5M2EXT",
-       {FPEncodingWrap::IEEE754,    FPEncodingWrap::E5M2,       OpFConvert});
-   add("ConvertBF16ToE4M3EXT",
-       {FPEncodingWrap::BF16,       FPEncodingWrap::E4M3,       OpFConvert});
-   add("ConvertBF16ToE5M2EXT",
-       {FPEncodingWrap::BF16,       FPEncodingWrap::E5M2,       OpFConvert});
-
-   add("ConvertInt4ToE4M3INTEL",
-       {FPEncodingWrap::Integer,    FPEncodingWrap::E4M3,       OpConvertSToF});
-   add("ConvertInt4ToE5M2INTEL",
-       {FPEncodingWrap::Integer,    FPEncodingWrap::E5M2,       OpConvertSToF});
-   add("ConvertInt4ToFP16INTEL",
-       {FPEncodingWrap::Integer,    FPEncodingWrap::IEEE754,    OpConvertSToF});
-   add("ConvertInt4ToBF16INTEL",
-       {FPEncodingWrap::Integer,    FPEncodingWrap::BF16,       OpConvertSToF});
-   add("ConvertFP16ToInt4INTEL",
-       {FPEncodingWrap::IEEE754,    FPEncodingWrap::Integer,    OpConvertFToS});
-   add("ConvertBF16ToInt4INTEL",
-       {FPEncodingWrap::BF16,       FPEncodingWrap::Integer,    OpConvertFToS});
+  // 4-bit conversions
+  add("ConvertE2M1ToE4M3INTEL",
+      {FPEncodingWrap::E2M1,      FPEncodingWrap::E4M3,         OpFConvert});
+  add("ConvertE2M1ToE5M2INTEL",
+      {FPEncodingWrap::E2M1,      FPEncodingWrap::E5M2,         OpFConvert});
+  add("ConvertE2M1ToFP16INTEL",
+      {FPEncodingWrap::E2M1,      FPEncodingWrap::IEEE754,      OpFConvert});
+  add("ConvertE2M1ToBF16INTEL",
+      {FPEncodingWrap::E2M1,      FPEncodingWrap::BF16,         OpFConvert});
+
+  add("ConvertInt4ToE4M3INTEL",
+      {FPEncodingWrap::Integer,      FPEncodingWrap::E4M3,      OpConvertSToF});
+  add("ConvertInt4ToE5M2INTEL",
+      {FPEncodingWrap::Integer,      FPEncodingWrap::E5M2,      OpConvertSToF});
+  add("ConvertInt4ToFP16INTEL",
+      {FPEncodingWrap::Integer,      FPEncodingWrap::IEEE754,   OpConvertSToF});
+  add("ConvertInt4ToBF16INTEL",
+      {FPEncodingWrap::Integer,      FPEncodingWrap::BF16,      OpConvertSToF});
+  add("ConvertInt4ToInt8INTEL",
+      {FPEncodingWrap::Integer,      FPEncodingWrap::Integer,   OpSConvert});
+
+  add("ConvertFP16ToE2M1INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E2M1,      OpFConvert});
+  add("ConvertBF16ToE2M1INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E2M1,      OpFConvert});
+  add("ConvertFP16ToInt4INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::Integer,   OpConvertFToS});
+  add("ConvertBF16ToInt4INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::Integer,   OpConvertFToS});
+
+  // 8-bit conversions
+  add("ConvertE4M3ToFP16EXT",
+      {FPEncodingWrap::E4M3,         FPEncodingWrap::IEEE754,      OpFConvert});
+  add("ConvertE5M2ToFP16EXT",
+      {FPEncodingWrap::E5M2,         FPEncodingWrap::IEEE754,      OpFConvert});
+  add("ConvertE4M3ToBF16EXT",
+      {FPEncodingWrap::E4M3,         FPEncodingWrap::BF16,         OpFConvert});
+  add("ConvertE5M2ToBF16EXT",
+      {FPEncodingWrap::E5M2,         FPEncodingWrap::BF16,         OpFConvert});
+  add("ConvertFP16ToE4M3EXT",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E4M3,         OpFConvert});
+  add("ConvertFP16ToE5M2EXT",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E5M2,         OpFConvert});
+  add("ConvertBF16ToE4M3EXT",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E4M3,         OpFConvert});
+  add("ConvertBF16ToE5M2EXT",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E5M2,         OpFConvert});
+
+  // SPV_INTEL_fp_conversions
+  add("ClampConvertFP16ToE2M1INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E2M1,
+       internal::OpClampConvertFToFINTEL});
+  add("ClampConvertBF16ToE2M1INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E2M1,
+       internal::OpClampConvertFToFINTEL});
+  add("ClampConvertFP16ToE4M3INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E4M3,
+       internal::OpClampConvertFToFINTEL});
+  add("ClampConvertBF16ToE4M3INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E4M3,
+       internal::OpClampConvertFToFINTEL});
+  add("ClampConvertFP16ToE5M2INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E5M2,
+       internal::OpClampConvertFToFINTEL});
+  add("ClampConvertBF16ToE5M2INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E5M2,
+       internal::OpClampConvertFToFINTEL});
+  add("ClampConvertFP16ToInt4INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::Integer,
+       internal::OpClampConvertFToSINTEL});
+  add("ClampConvertBF16ToInt4INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::Integer,
+       internal::OpClampConvertFToSINTEL});
+
+  add("StochasticRoundFP16ToE5M2INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E5M2,
+       internal::OpStochasticRoundFToFINTEL});
+  add("StochasticRoundFP16ToE4M3INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E4M3,
+       internal::OpStochasticRoundFToFINTEL});
+  add("StochasticRoundBF16ToE5M2INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E5M2,
+       internal::OpStochasticRoundFToFINTEL});
+  add("StochasticRoundBF16ToE4M3INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E4M3,
+       internal::OpStochasticRoundFToFINTEL});
+  add("StochasticRoundFP16ToE2M1INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E2M1,
+       internal::OpStochasticRoundFToFINTEL});
+  add("StochasticRoundBF16ToE2M1INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E2M1,
+       internal::OpStochasticRoundFToFINTEL});
+  add("ClampStochasticRoundFP16ToInt4INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::Integer,
+       internal::OpClampStochasticRoundFToSINTEL});
+  add("ClampStochasticRoundBF16ToInt4INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::Integer,
+       internal::OpClampStochasticRoundFToSINTEL});
+
+  add("ClampStochasticRoundFP16ToE5M2INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E5M2,
+       internal::OpClampStochasticRoundFToFINTEL});
+  add("ClampStochasticRoundFP16ToE4M3INTEL",
+      {FPEncodingWrap::IEEE754,      FPEncodingWrap::E4M3,
+       internal::OpClampStochasticRoundFToFINTEL});
+  add("ClampStochasticRoundBF16ToE5M2INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E5M2,
+       internal::OpClampStochasticRoundFToFINTEL});
+  add("ClampStochasticRoundBF16ToE4M3INTEL",
+      {FPEncodingWrap::BF16,         FPEncodingWrap::E4M3,
+       internal::OpClampStochasticRoundFToFINTEL});
 }
 
 // clang-format on