merge amd-develop into amd-staging

Author: ronlieb

include/LLVMSPIRVExtensions.inc

@@ -82,4 +82,5 @@ EXT(SPV_INTEL_ternary_bitwise_function)
 EXT(SPV_INTEL_int4)
 EXT(SPV_INTEL_function_variants)
 EXT(SPV_INTEL_shader_atomic_bfloat16)
+EXT(SPV_EXT_float8)
 EXT(SPV_INTEL_predicated_io)

include/LLVMSPIRVLib.h

@@ -43,8 +43,11 @@
 
 #include "LLVMSPIRVOpts.h"
 
+#include <cstdint>
 #include <iostream>
+#include <optional>
 #include <string>
+#include <vector>
 
 namespace llvm {
 // Pass initialization functions need to be declared before inclusion of

include/LLVMSPIRVOpts.h

@@ -47,6 +47,7 @@
 #include <map>
 #include <optional>
 #include <unordered_map>
+#include <vector>
 
 namespace llvm {
 class IntrinsicInst;

lib/SPIRV/OCLToSPIRV.cpp

@@ -64,8 +64,8 @@ using namespace OCLUtil;
 namespace SPIRV {
 static size_t getOCLCpp11AtomicMaxNumOps(StringRef Name) {
   return StringSwitch<size_t>(Name)
-      .Cases("load", "flag_test_and_set", "flag_clear", 3)
-      .Cases("store", "exchange", 4)
+      .Cases({"load", "flag_test_and_set", "flag_clear"}, 3)
+      .Cases({"store", "exchange"}, 4)
       .StartsWith("compare_exchange", 6)
       .StartsWith("fetch", 4)
       .Default(0);

lib/SPIRV/SPIRVInternal.h

@@ -374,6 +374,7 @@ const static char ConvertHandleToImageINTEL[] = "ConvertHandleToImageINTEL";
 const static char ConvertHandleToSamplerINTEL[] = "ConvertHandleToSamplerINTEL";
 const static char ConvertHandleToSampledImageINTEL[] =
     "ConvertHandleToSampledImageINTEL";
+const static char InternalBuiltinPrefix[] = "__builtin_spirv_";
 } // namespace kSPIRVName
 
 namespace kSPIRVPostfix {
@@ -666,7 +667,7 @@ Op getSPIRVFuncOC(StringRef Name, SmallVectorImpl<std::string> *Dec = nullptr);
 bool getSPIRVBuiltin(const std::string &Name, spv::BuiltIn &Builtin);
 
 /// \param Name LLVM function name
-/// \param DemangledName demanged name of the OpenCL built-in function
+/// \param DemangledName demangled name of the OpenCL built-in function
 /// \returns true if Name is the name of the OpenCL built-in function,
 /// false for other functions
 bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp = false);
@@ -729,6 +730,9 @@ CallInst *addCallInst(Module *M, StringRef FuncName, Type *RetTy,
                       StringRef InstName = SPIR_TEMP_NAME_PREFIX_CALL,
                       bool TakeFuncName = true);
 
+/// Check if an LLVM type is spirv.CooperativeMatrixKHR.
+bool isLLVMCooperativeMatrixType(llvm::Type *Ty);
+
 /// Add a call instruction for SPIR-V builtin function.
 CallInst *addCallInstSPIRV(Module *M, StringRef FuncName, Type *RetTy,
                            ArrayRef<Value *> Args, AttributeList *Attrs,
@@ -1030,6 +1034,84 @@ bool postProcessBuiltinsReturningStruct(Module *M, bool IsCpp = false);
 
 bool postProcessBuiltinsWithArrayArguments(Module *M, bool IsCpp = false);
 
+/// \param MangledName LLVM function name.
+/// \param DemangledName demangled name of the input function if it is the
+/// translator's internal built-in function.
+/// \returns true if MangledName is the name of the translator's internal
+/// built-in function, false for other functions.
+/// Used for 'mini'-floats conversion functions
+bool isInternalSPIRVBuiltin(StringRef MangledName, StringRef &DemangledName);
+
+// Wrapper around SPIR-V 1.6.4 FP Encoding to be used in the conversion
+// descriptor
+enum FPEncodingWrap {
+  Integer = FPEncoding::FPEncodingMax - 1,
+  IEEE754 = FPEncoding::FPEncodingMax,
+  BF16 = FPEncoding::FPEncodingBFloat16KHR,
+  E4M3 = FPEncoding::FPEncodingFloat8E4M3EXT,
+  E5M2 = FPEncoding::FPEncodingFloat8E5M2EXT,
+};
+
+// Structure describing non-trivial conversions (FP8 and int4)
+struct FPConversionDesc {
+  FPEncodingWrap SrcEncoding;
+  FPEncodingWrap DstEncoding;
+  SPIRVWord ConvOpCode;
+
+  // To use as a key in std::map
+  bool operator==(const FPConversionDesc &Other) const {
+    return SrcEncoding == Other.SrcEncoding &&
+           DstEncoding == Other.DstEncoding && ConvOpCode == Other.ConvOpCode;
+  }
+
+  bool operator<(const FPConversionDesc &Other) const {
+    if (ConvOpCode != Other.ConvOpCode)
+      return ConvOpCode < Other.ConvOpCode;
+    if (SrcEncoding != Other.SrcEncoding)
+      return SrcEncoding < Other.SrcEncoding;
+    return DstEncoding < Other.DstEncoding;
+  }
+};
+
+// Maps internal builtin name to conversion descriptor
+typedef SPIRVMap<llvm::StringRef, FPConversionDesc> FPConvertToEncodingMap;
+
+// clang-format off
+template <> inline void FPConvertToEncodingMap::init() {
+   // 8-bit conversions
+   add("ConvertE4M3ToFP16EXT",
+       {FPEncodingWrap::E4M3,       FPEncodingWrap::IEEE754,    OpFConvert});
+   add("ConvertE5M2ToFP16EXT",
+       {FPEncodingWrap::E5M2,       FPEncodingWrap::IEEE754,    OpFConvert});
+   add("ConvertE4M3ToBF16EXT",
+       {FPEncodingWrap::E4M3,       FPEncodingWrap::BF16,       OpFConvert});
+   add("ConvertE5M2ToBF16EXT",
+       {FPEncodingWrap::E5M2,       FPEncodingWrap::BF16,       OpFConvert});
+   add("ConvertFP16ToE4M3EXT",
+       {FPEncodingWrap::IEEE754,    FPEncodingWrap::E4M3,       OpFConvert});
+   add("ConvertFP16ToE5M2EXT",
+       {FPEncodingWrap::IEEE754,    FPEncodingWrap::E5M2,       OpFConvert});
+   add("ConvertBF16ToE4M3EXT",
+       {FPEncodingWrap::BF16,       FPEncodingWrap::E4M3,       OpFConvert});
+   add("ConvertBF16ToE5M2EXT",
+       {FPEncodingWrap::BF16,       FPEncodingWrap::E5M2,       OpFConvert});
+
+   add("ConvertInt4ToE4M3INTEL",
+       {FPEncodingWrap::Integer,    FPEncodingWrap::E4M3,       OpConvertSToF});
+   add("ConvertInt4ToE5M2INTEL",
+       {FPEncodingWrap::Integer,    FPEncodingWrap::E5M2,       OpConvertSToF});
+   add("ConvertInt4ToFP16INTEL",
+       {FPEncodingWrap::Integer,    FPEncodingWrap::IEEE754,    OpConvertSToF});
+   add("ConvertInt4ToBF16INTEL",
+       {FPEncodingWrap::Integer,    FPEncodingWrap::BF16,       OpConvertSToF});
+   add("ConvertFP16ToInt4INTEL",
+       {FPEncodingWrap::IEEE754,    FPEncodingWrap::Integer,    OpConvertFToS});
+   add("ConvertBF16ToInt4INTEL",
+       {FPEncodingWrap::BF16,       FPEncodingWrap::Integer,    OpConvertFToS});
+}
+
+// clang-format on
+
 } // namespace SPIRV
 
 #endif // SPIRV_SPIRVINTERNAL_H

lib/SPIRV/SPIRVReader.cpp

@@ -298,6 +298,9 @@ std::optional<uint64_t> SPIRVToLLVM::getAlignment(SPIRVValue *V) {
 
 Type *SPIRVToLLVM::transFPType(SPIRVType *T) {
   switch (T->getFloatBitWidth()) {
+  case 8:
+    // No LLVM IR counter part for FP8 - map it on i8
+    return Type::getIntNTy(*Context, 8);
   case 16:
     if (T->isTypeFloat(16, FPEncodingBFloat16KHR))
       return Type::getBFloatTy(*Context);
@@ -1060,6 +1063,22 @@ Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F,
   CastInst::CastOps CO = Instruction::BitCast;
   bool IsExt =
       Dst->getScalarSizeInBits() > Src->getType()->getScalarSizeInBits();
+
+  auto GetFPEncoding = [](SPIRVType *Ty) -> FPEncodingWrap {
+    if (Ty->isTypeFloat()) {
+      unsigned Enc =
+          static_cast<SPIRVTypeFloat *>(Ty)->getFloatingPointEncoding();
+      return static_cast<FPEncodingWrap>(Enc);
+    }
+    if (Ty->isTypeInt())
+      return FPEncodingWrap::Integer;
+    return FPEncodingWrap::IEEE754;
+  };
+
+  auto IsFP8Encoding = [](FPEncodingWrap Encoding) -> bool {
+    return Encoding == FPEncodingWrap::E4M3 || Encoding == FPEncodingWrap::E5M2;
+  };
+
   switch (BC->getOpCode()) {
   case OpPtrCastToGeneric:
   case OpGenericCastToPtr:
@@ -1081,9 +1100,56 @@ Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F,
   case OpUConvert:
     CO = IsExt ? Instruction::ZExt : Instruction::Trunc;
     break;
-  case OpFConvert:
-    CO = IsExt ? Instruction::FPExt : Instruction::FPTrunc;
+  case OpConvertSToF:
+  case OpConvertFToS:
+  case OpConvertUToF:
+  case OpConvertFToU:
+  case OpFConvert: {
+    const auto OC = BC->getOpCode();
+    {
+      auto SPVOps = BC->getOperands();
+      auto *SPVSrcTy = SPVOps[0]->getType();
+      auto *SPVDstTy = BC->getType();
+
+      auto GetEncodingAndUpdateType =
+          [GetFPEncoding](SPIRVType *&SPVTy) -> FPEncodingWrap {
+        if (SPVTy->isTypeVector()) {
+          SPVTy = SPVTy->getVectorComponentType();
+        } else if (SPVTy->isTypeCooperativeMatrixKHR()) {
+          auto *MT = static_cast<SPIRVTypeCooperativeMatrixKHR *>(SPVTy);
+          SPVTy = MT->getCompType();
+        }
+        return GetFPEncoding(SPVTy);
+      };
+
+      FPEncodingWrap SrcEnc = GetEncodingAndUpdateType(SPVSrcTy);
+      FPEncodingWrap DstEnc = GetEncodingAndUpdateType(SPVDstTy);
+      if (IsFP8Encoding(SrcEnc) || IsFP8Encoding(DstEnc) ||
+          SPVSrcTy->isTypeInt(4) || SPVDstTy->isTypeInt(4)) {
+        FPConversionDesc FPDesc = {SrcEnc, DstEnc,
+                                   static_cast<SPIRVWord>(BC->getOpCode())};
+        auto Conv = SPIRV::FPConvertToEncodingMap::rmap(FPDesc);
+        std::vector<Value *> Ops = {Src};
+        std::vector<Type *> OpsTys = {Src->getType()};
+
+        std::string BuiltinName =
+            kSPIRVName::InternalBuiltinPrefix + std::string(Conv);
+        BuiltinFuncMangleInfo Info;
+        std::string MangledName = mangleBuiltin(BuiltinName, OpsTys, &Info);
+
+        FunctionType *FTy = FunctionType::get(Dst, OpsTys, false);
+        FunctionCallee Func = M->getOrInsertFunction(MangledName, FTy);
+        return CallInst::Create(Func, Ops, "", BB);
+      }
+    }
+
+    if (OC == OpFConvert) {
+      CO = IsExt ? Instruction::FPExt : Instruction::FPTrunc;
+      break;
+    }
+    CO = static_cast<CastInst::CastOps>(OpCodeMap::rmap(OC));
     break;
+  }
   case OpBitcast:
     if (Src->getType()->isPointerTy() && Dst->isPointerTy()) {
       if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) {
@@ -1093,7 +1159,8 @@ Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F,
         else
           return Src; // Spuriously inserted pointer BC.
       }
-    } else if (Src->getType() == Dst) { // Spuriously inserted BC
+    } else if ((!Dst->isPointerTy() && Dst == Src->getType()) ||
+               (Src->getType() == Dst)) { // Spuriously inserted BC
       return Src;
     } else {
       // OpBitcast need to be handled as a special-case when the source is a
@@ -3037,11 +3104,29 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
     if (isCvtOpCode(OC) && OC != OpGenericCastToPtrExplicit) {
       auto *BI = static_cast<SPIRVInstruction *>(BV);
       Value *Inst = nullptr;
-      if (BI->hasFPRoundingMode() || BI->isSaturatedConversion() ||
-          BI->getType()->isTypeCooperativeMatrixKHR())
+      if (BI->hasFPRoundingMode() || BI->isSaturatedConversion()) {
         Inst = transSPIRVBuiltinFromInst(BI, BB);
-      else
+      } else if (BI->getType()->isTypeCooperativeMatrixKHR()) {
+        // For cooperative matrix conversions generate __builtin_spirv
+        // conversions instead of __spirv_FConvert in case of mini-float
+        // type element type.
+        auto *OutMatrixElementTy =
+            static_cast<SPIRVTypeCooperativeMatrixKHR *>(BI->getType())
+                ->getCompType();
+        auto *InMatrixElementTy =
+            static_cast<SPIRVTypeCooperativeMatrixKHR *>(
+                static_cast<SPIRVUnary *>(BI)->getOperand(0)->getType())
+                ->getCompType();
+        if (OutMatrixElementTy->isTypeFloat(8, FPEncodingFloat8E4M3EXT) ||
+            OutMatrixElementTy->isTypeFloat(8, FPEncodingFloat8E5M2EXT) ||
+            InMatrixElementTy->isTypeFloat(8, FPEncodingFloat8E4M3EXT) ||
+            InMatrixElementTy->isTypeFloat(8, FPEncodingFloat8E5M2EXT))
+          Inst = transConvertInst(BV, F, BB);
+        else
+          Inst = transSPIRVBuiltinFromInst(BI, BB);
+      } else {
         Inst = transConvertInst(BV, F, BB);
+      }
       return mapValue(BV, Inst);
     }
     return mapValue(

lib/SPIRV/SPIRVToOCL.cpp

@@ -679,6 +679,13 @@ void SPIRVToOCLBase::visitCallGenericCastToPtrExplicitBuiltIn(CallInst *CI,
 
 void SPIRVToOCLBase::visitCallSPIRVCvtBuiltin(CallInst *CI, Op OC,
                                               StringRef DemangledName) {
+  if (auto *TET =
+          dyn_cast<TargetExtType>(CI->getFunctionType()->getReturnType())) {
+    // Preserve any cooperative matrix type conversions as SPIR-V calls.
+    if (TET->getName() == "spirv.CooperativeMatrixKHR") {
+      return;
+    }
+  }
   std::string CastBuiltInName;
   if (isCvtFromUnsignedOpCode(OC))
     CastBuiltInName = "u";

lib/SPIRV/SPIRVTypeScavenger.cpp

@@ -453,6 +453,7 @@ bool SPIRVTypeScavenger::typeIntrinsicCall(
     case OpAtomicLoad:
     case OpAtomicExchange:
     case OpAtomicCompareExchange:
+    case OpAtomicCompareExchangeWeak:
     case OpAtomicIAdd:
     case OpAtomicISub:
     case OpAtomicFAddEXT:

lib/SPIRV/SPIRVUtil.cpp

@@ -40,6 +40,7 @@
 
 // This file needs to be included before anything that declares
 // llvm::PointerType to avoid a compilation bug on MSVC.
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/Demangle/ItaniumDemangle.h"
 
 #include "FunctionDescriptor.h"
@@ -267,6 +268,12 @@ bool isSYCLBfloat16Type(llvm::Type *Ty) {
   return false;
 }
 
+bool isLLVMCooperativeMatrixType(llvm::Type *Ty) {
+  if (auto *TargetTy = dyn_cast<TargetExtType>(Ty))
+    return TargetTy->getName() == "spirv.CooperativeMatrixKHR";
+  return false;
+}
+
 Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef<Type *> ArgTypes,
                               StringRef Name, BuiltinFuncMangleInfo *Mangle,
                               AttributeList *Attrs, bool TakeName) {
@@ -442,7 +449,7 @@ bool getSPIRVBuiltin(const std::string &OrigName, spv::BuiltIn &B) {
   return getByName(R.str(), B);
 }
 
-// Demangled name is a substring of the name. The DemangledName is updated only
+// DemangledName is a substring of Name. The DemangledName is updated only
 // if true is returned
 bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp) {
   if (Name == "printf") {
@@ -487,6 +494,21 @@ bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp) {
   return false;
 }
 
+// DemangledName is a substring of Name. The DemangledName is updated only
+// if true is returned.
+bool isInternalSPIRVBuiltin(StringRef Name, StringRef &DemangledName) {
+  if (!Name.starts_with("_Z"))
+    return false;
+  constexpr unsigned DemangledNameLenStart = 2;
+  size_t Start = Name.find_first_not_of("0123456789", DemangledNameLenStart);
+  if (!Name.substr(Start, Name.size() - 1)
+           .starts_with(kSPIRVName::InternalBuiltinPrefix))
+    return false;
+  DemangledName = llvm::itaniumDemangle(Name.data(), false);
+  DemangledName.consume_front(kSPIRVName::InternalBuiltinPrefix);
+  return true;
+}
+
 // Check if a mangled type Name is unsigned
 bool isMangledTypeUnsigned(char Mangled) {
   return Mangled == 'h'    /* uchar */
@@ -601,11 +623,11 @@ static std::string demangleBuiltinOpenCLTypeName(StringRef MangledStructName) {
 /// floating point type.
 static Type *parsePrimitiveType(LLVMContext &Ctx, StringRef Name) {
   return StringSwitch<Type *>(Name)
-      .Cases("char", "signed char", "unsigned char", Type::getInt8Ty(Ctx))
-      .Cases("short", "unsigned short", Type::getInt16Ty(Ctx))
-      .Cases("int", "unsigned int", Type::getInt32Ty(Ctx))
-      .Cases("long", "unsigned long", Type::getInt64Ty(Ctx))
-      .Cases("long long", "unsigned long long", Type::getInt64Ty(Ctx))
+      .Cases({"char", "signed char", "unsigned char"}, Type::getInt8Ty(Ctx))
+      .Cases({"short", "unsigned short"}, Type::getInt16Ty(Ctx))
+      .Cases({"int", "unsigned int"}, Type::getInt32Ty(Ctx))
+      .Cases({"long", "unsigned long"}, Type::getInt64Ty(Ctx))
+      .Cases({"long long", "unsigned long long"}, Type::getInt64Ty(Ctx))
       .Case("half", Type::getHalfTy(Ctx))
       .Case("float", Type::getFloatTy(Ctx))
       .Case("double", Type::getDoubleTy(Ctx))