Handle canonical types in constraints checks (#129278)

Fixes #126604

First commit is a revert of the revert.

The second commit is an update to the casting logic so that we can have
both casting logic that treats `__Canon` as a `class __Canon : object`
and nothing else (this is compatible with how casting in CoreCLR VM
works), but also another casting logic that treats `__Canon` as "some
reference type, could be something that does cast". Then update
constraints check to use this flavor of the casting logic.

Author: MichalStrehovsky

src/coreclr/nativeaot/System.Private.TypeLoader/src/System.Private.TypeLoader.csproj

@@ -120,6 +120,9 @@
     <Compile Include="$(CompilerCommonPath)\TypeSystem\Common\CastingHelper.cs">
       <Link>Internal\TypeSystem\CastingHelper.cs</Link>
     </Compile>
+    <Compile Include="$(CompilerCommonPath)\TypeSystem\Canon\CastingHelper.Canon.cs">
+      <Link>Internal\TypeSystem\CastingHelper.Canon.cs</Link>
+    </Compile>
     <Compile Include="$(CompilerCommonPath)\TypeSystem\Common\DefType.cs">
       <Link>Internal\TypeSystem\DefType.cs</Link>
     </Compile>

src/coreclr/tools/Common/TypeSystem/Canon/CastingHelper.Canon.cs

@@ -0,0 +1,93 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Internal.TypeSystem
+{
+    internal sealed class CanonicalTypeCastingHandler : ICanonicalTypeCastingHandler
+    {
+        /// <summary>
+        /// Check if <paramref name="otherType"/> is a canonical type that <paramref name="thisType"/>
+        /// can be cast to. __Canon accepts any reference type; __UniversalCanon accepts any type.
+        /// Pointers, byrefs, and function pointers are not valid instantiation arguments.
+        /// </summary>
+        public static bool IsCanonicalCastTarget(TypeDesc thisType, TypeDesc otherType)
+        {
+            TypeSystemContext context = thisType.Context;
+
+            if (context.IsCanonicalDefinitionType(otherType, CanonicalFormKind.Universal))
+                return true;
+
+            if (context.IsCanonicalDefinitionType(otherType, CanonicalFormKind.Specific))
+                return thisType.IsGCPointer;
+
+            return false;
+        }
+
+        /// <summary>
+        /// Check if two type arguments can be considered matching because one (or both) is canonical.
+        /// __Canon matches any reference type; __UniversalCanon matches any type.
+        /// </summary>
+        public static bool IsCanonicalTypeArgMatch(TypeDesc type, TypeDesc otherType)
+        {
+            TypeSystemContext context = type.Context;
+
+            if (context.IsCanonicalDefinitionType(otherType, CanonicalFormKind.Universal))
+                return true;
+
+            if (context.IsCanonicalDefinitionType(otherType, CanonicalFormKind.Specific))
+                return type.IsGCPointer || context.IsCanonicalDefinitionType(type, CanonicalFormKind.Any);
+
+            if (context.IsCanonicalDefinitionType(type, CanonicalFormKind.Universal))
+                return true;
+
+            if (context.IsCanonicalDefinitionType(type, CanonicalFormKind.Specific))
+                return otherType.IsGCPointer || context.IsCanonicalDefinitionType(otherType, CanonicalFormKind.Any);
+
+            // For non-leaf types (e.g., Arg2<string> vs Arg2<__Canon>), check if they are
+            // canon-equivalent: same type definition with canon-compatible type arguments.
+            if (IsCanonEquivalent(type, otherType))
+                return true;
+
+            // For parameterized types like arrays (string[] vs __Canon[]),
+            // recursively match the element/parameter type.
+            if (type is ParameterizedType paramType && otherType is ParameterizedType otherParamType
+                && type.Category == otherType.Category)
+            {
+                if (type is ArrayType arrayType && otherType is ArrayType otherArrayType
+                    && arrayType.Rank != otherArrayType.Rank)
+                    return false;
+
+                return IsCanonicalTypeArgMatch(paramType.ParameterType, otherParamType.ParameterType);
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Check if two types are equivalent considering canonical type matching rules.
+        /// Same type definition with all type arguments either equal or canon-compatible.
+        /// </summary>
+        public static bool IsCanonEquivalent(TypeDesc thisType, TypeDesc otherType)
+        {
+            if (!thisType.HasSameTypeDefinition(otherType))
+                return false;
+
+            Instantiation thisInst = thisType.Instantiation;
+            Instantiation otherInst = otherType.Instantiation;
+
+            if (thisInst.Length == 0)
+                return false;
+
+            for (int i = 0; i < thisInst.Length; i++)
+            {
+                if (thisInst[i] == otherInst[i])
+                    continue;
+
+                if (!IsCanonicalTypeArgMatch(thisInst[i], otherInst[i]))
+                    return false;
+            }
+
+            return true;
+        }
+    }
+}

src/coreclr/tools/Common/TypeSystem/Canon/TypeSystemConstraintsHelpers.Canon.cs

@@ -0,0 +1,48 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+
+namespace Internal.TypeSystem
+{
+    public static partial class TypeSystemConstraintsHelpers
+    {
+        private static bool IsSpecialTypeMeetingConstraint(TypeDesc type, GenericConstraints constraint)
+        {
+            TypeSystemContext context = type.Context;
+
+            return constraint switch
+            {
+                GenericConstraints.ReferenceTypeConstraint => context.IsCanonicalDefinitionType(type, CanonicalFormKind.Any),
+                GenericConstraints.DefaultConstructorConstraint => context.IsCanonicalDefinitionType(type, CanonicalFormKind.Any),
+                GenericConstraints.NotNullableValueTypeConstraint => context.IsCanonicalDefinitionType(type, CanonicalFormKind.Universal),
+                _ => throw new UnreachableException()
+            };
+        }
+
+        /// <summary>
+        /// Checks if <paramref name="instantiationParam"/> can satisfy the type constraint
+        /// <paramref name="instantiatedConstraintType"/> when the param or constraint IS a canonical
+        /// definition type (__Canon or __UniversalCanon). Handles wildcard semantics only;
+        /// structural matching (interface walking, base chain, variance) is in CastingHelper.
+        /// </summary>
+        private static bool CanCastToConstraintWithCanon(TypeDesc instantiationParam, TypeDesc instantiatedConstraintType)
+        {
+            TypeSystemContext context = instantiationParam.Context;
+
+            // If the instantiation param is a canonical definition type (__Canon or __UniversalCanon),
+            // it acts as a wildcard — any concrete type substituted at runtime will be validated then.
+            if (context.IsCanonicalDefinitionType(instantiationParam, CanonicalFormKind.Any))
+                return true;
+
+            // If the constraint type itself is a canonical definition type, check compatibility directly.
+            // E.g., "where T : U" with U=__Canon means T must be a plausible match for __Canon.
+            if (context.IsCanonicalDefinitionType(instantiatedConstraintType, CanonicalFormKind.Universal))
+                return true;
+            if (context.IsCanonicalDefinitionType(instantiatedConstraintType, CanonicalFormKind.Specific))
+                return instantiationParam.IsGCPointer;
+
+            return false;
+        }
+    }
+}

src/coreclr/tools/Common/TypeSystem/Common/CastingHelper.NonCanon.cs

@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Internal.TypeSystem
+{
+    internal sealed class CanonicalTypeCastingHandler : INonCanonicalTypeCastingHandler;
+}