Fixed NativeBoolean/NativeChar no longer being used for vtable function pointers.

Related: #147
Related: #200

Author: PathogenDavid

Biohazrd.CSharp/#Transformations/CSharpTranslationVerifier.cs

@@ -229,6 +229,11 @@ static bool TypeCanHaveDefaultInCSharp(TranslatedLibrary library, TypeReference
                                     return TypeCanHaveDefaultInCSharp(library, typedef.UnderlyingType);
                                 case TranslatedEnum:
                                     return true;
+                                // NativeBoolean and NativeChar will become bool/char on the trampoline surface and should not appear in places where trampolines aren't generated since
+                                // they end up using MarshalAs instead.
+                                case NativeBooleanDeclaration:
+                                case NativeCharDeclaration:
+                                    return true;
                             }
                         }
 

Biohazrd.CSharp/#Transformations/WrapNonBlittableTypesWhereNecessaryTransformation.cs

@@ -66,8 +66,17 @@ protected override TypeTransformationResult TransformCSharpBuiltinTypeReference(
             if (context.Parent is PointerTypeReference)
             { return type; }
 
+            // If the type is for a virtual method return value, we always wrap non-blittable types since these types will be used for the vtable pointer
+            // See https://github.com/InfectedLibraries/Biohazrd/issues/200 for details
+            if (context.ParentDeclaration is TranslatedFunction { IsVirtual: true })
+            { }
+            // Same for parameters of virtual methods
+            //TODO: C# 10: Use list pattern syntax: context.ParentDeclarations is [.., TranslatedParameter, TranslatedFunction { IsVirtual: True }]
+            else if (context.ParentDeclaration is TranslatedParameter
+                && context.ParentDeclarations.Length >= 2 && context.ParentDeclarations[context.ParentDeclarations.Length - 2] is TranslatedFunction { IsVirtual: true })
+            { }
             // If the parent type reference isn't a function pointer, Blittablebool/BlittableChar should not be necessary
-            if (!context.Parents.Any(t => t is FunctionPointerTypeReference))
+            else if (!context.Parents.Any(t => t is FunctionPointerTypeReference))
             { return type; }
 
             if (type.Type == CSharpBuiltinType.Bool)

Biohazrd.CSharp/CSharpLibraryGenerator.Constants.cs

@@ -18,7 +18,19 @@ private string GetConstantAsString(VisitorContext context, TranslatedDeclaration
                 case IntegerConstant integerConstant:
                 {
                     // Bools come through as integer types
-                    if (targetType is CSharpBuiltinTypeReference cSharpTypeReference && cSharpTypeReference.Type == CSharpBuiltinType.Bool)
+                    static bool IsBooleanType(VisitorContext context, TypeReference targetType)
+                    {
+                        if (targetType is CSharpBuiltinTypeReference cSharpTypeReference && cSharpTypeReference.Type == CSharpBuiltinType.Bool)
+                        { return true; }
+                        // NativeBoolean is used for virtual methods which end up getting wrapped by trampolines which use bool so emitting them as true/false contants are valid in that context
+                        // See https://github.com/InfectedLibraries/Biohazrd/issues/200 for details.
+                        else if (targetType is TranslatedTypeReference translatedTypeReference && translatedTypeReference.TryResolve(context.Library) is NativeBooleanDeclaration)
+                        { return true; }
+                        else
+                        { return false; }
+                    }
+
+                    if (IsBooleanType(context, targetType))
                     {
                         if (integerConstant.Value == 0)
                         { return "false"; }

Biohazrd.CSharp/CSharpLibraryGenerator.Functions.cs

@@ -189,7 +189,7 @@ private void EmitFunctionTrampoline(VisitorContext context, EmitFunctionContext
             Writer.Write($"{declaration.Accessibility.ToCSharpKeyword()} unsafe ");
             if (!declaration.IsInstanceMethod)
             { Writer.Write("static "); }
-            WriteType(context, declaration, declaration.ReturnType);
+            WriteTypeForTrampoline(context, declaration, declaration.ReturnType);
             Writer.Write($" {SanitizeIdentifier(declaration.Name)}(");
             EmitFunctionParameterList(context, emitContext, declaration, EmitParameterListMode.TrampolineParameters);
             Writer.WriteLine(')');
@@ -406,7 +406,10 @@ void WriteOutReturnBuffer(EmitFunctionContext emitContext)
                         if (mode == EmitParameterListMode.DllImportParameters && parameter.Type.IsCSharpType(CSharpBuiltinType.Bool))
                         { Writer.Write("[MarshalAs(UnmanagedType.I1)] "); }
 
-                        WriteType(parameterContext, parameter, parameter.Type);
+                        if (mode == EmitParameterListMode.TrampolineParameters)
+                        { WriteTypeForTrampoline(parameterContext, parameter, parameter.Type); }
+                        else
+                        { WriteType(parameterContext, parameter, parameter.Type); }
                     }
 
                     if (writeNames)
@@ -469,6 +472,26 @@ private void EmitMethodImplAttribute(TranslatedFunction declaration)
             Writer.WriteLine(")]");
         }
 
+        private void WriteTypeForTrampoline(VisitorContext context, TranslatedDeclaration declaration, TypeReference type)
+        {
+            // For trampolines we want to hide the semantics of NativeBoolean/NativeChar so we silently replace them with their actual C# type
+            // See https://github.com/InfectedLibraries/Biohazrd/issues/200 for details.
+            if (type is TranslatedTypeReference typeReference)
+            {
+                switch (typeReference.TryResolve(context.Library))
+                {
+                    case NativeBooleanDeclaration:
+                        WriteType(context, declaration, CSharpBuiltinType.Bool);
+                        return;
+                    case NativeCharDeclaration:
+                        WriteType(context, declaration, CSharpBuiltinType.Char);
+                        return;
+                }
+            }
+
+            WriteType(context, declaration, type);
+        }
+
         protected override void VisitParameter(VisitorContext context, TranslatedParameter declaration)
             => FatalContext(context, declaration);
     }