cross-platform wrapper around memory-backed memorymappedfile creation (#8403)

* cross-platform wrapper around memory-backed memorymappedfile creation

* reenable FCS tests on non-windows platforms

* publish test results on non-windows platforms

* ensure correct .net sdk versions are installed regardless of VM image preloads

* simple workaround for missing mscorlib versions to TFMs

* publish per-framework xml results and report them in AZDO

* rework mono fixes to use ByteArrayMemory directly

* move runningOnMono call to bytes.fs and patch up callsites

* revert to build-only on mono-containing platforms

another MR will be required to fix the underlying logic before those can be changed to run tests again

* trigger ci

Author: baronfel

azure-pipelines.yml

@@ -342,6 +342,11 @@ stages:
           steps:
           - checkout: self
             clean: true
+          - task: UseDotNet@2
+            displayName: 'Use .NET Core sdk'
+            inputs:
+              useGlobalJson: true
+              workingDirectory: fcs
           - script: fcs\build.cmd TestAndNuget
             displayName: Build / Test
           - task: PublishTestResults@2
@@ -350,6 +355,7 @@ stages:
               testResultsFormat: 'NUnit'
               testResultsFiles: '*.xml'
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
+              publishRunAttachments: true
             continueOnError: true
             condition: always()
 
@@ -362,20 +368,48 @@ stages:
           steps:
           - checkout: self
             clean: true
+          - task: UseDotNet@2
+            displayName: 'Use .NET Core sdk'
+            inputs:
+              useGlobalJson: true
+              workingDirectory: fcs
           - script: ./fcs/build.sh
-            displayName: Build
+            displayName: Build / Test
+          - task: PublishTestResults@2
+            displayName: Publish Test Results
+            inputs:
+              testResultsFormat: 'NUnit'
+              testResultsFiles: '*.xml'
+              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
+              publishRunAttachments: true
+            continueOnError: true
+            condition: always()
 
-        # - job: MacOS_FCS
-        #   pool:
-        #     vmImage: macOS-latest
-        #   variables:
-        #   - name: _SignType
-        #     value: Test
-        #   steps:
-        #   - checkout: self
-        #     clean: true
-        #   - script: ./fcs/build.sh
-        #     displayName: Build
+        - job: MacOS_FCS
+          pool:
+            vmImage: macOS-latest
+          variables:
+          - name: _SignType
+            value: Test
+          steps:
+          - checkout: self
+            clean: true
+          - task: UseDotNet@2
+            displayName: 'Use .NET Core sdk'
+            inputs:
+              useGlobalJson: true
+              workingDirectory: fcs
+          - script: ./fcs/build.sh
+            displayName: Build / Test
+          - task: PublishTestResults@2
+            displayName: Publish Test Results
+            inputs:
+              testResultsFormat: 'NUnit'
+              testResultsFiles: '*.xml'
+              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
+              publishRunAttachments: true
+            continueOnError: true
+            condition: always()
 
 #---------------------------------------------------------------------------------------------------------------------#
 #                                                    Post Build                                                       #

eng/Versions.props

@@ -183,7 +183,7 @@
     <NUnitVersion>3.11.0</NUnitVersion>
     <NUnit3TestAdapterVersion>3.11.2</NUnit3TestAdapterVersion>
     <NUnitLiteVersion>3.11.0</NUnitLiteVersion>
-    <NunitXmlTestLoggerVersion>2.1.36</NunitXmlTestLoggerVersion>
+    <NunitXmlTestLoggerVersion>2.1.41</NunitXmlTestLoggerVersion>
     <RoslynToolsSignToolVersion>1.0.0-beta2-dev3</RoslynToolsSignToolVersion>
     <StrawberryPerlVersion>5.28.0.1</StrawberryPerlVersion>
     <StreamJsonRpcVersion>2.0.187</StreamJsonRpcVersion>

fcs/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj

@@ -1,5 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\netfx.props" />
+  <Import Project="..\..\eng\Versions.props"/>   <!-- keep our test deps in line with the overall compiler -->
   <PropertyGroup>
     <TargetFrameworks>$(FcsTargetNetFxFramework);netcoreapp3.0</TargetFrameworks>
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
@@ -60,6 +61,9 @@
     <Compile Include="$(FSharpSourcesRoot)\..\tests\service\TreeVisitorTests.fs">
       <Link>TreeVisitorTests.fs</Link>
     </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\..\tests\service\ScriptOptionsTests.fs">
+      <Link>ScriptOptionsTests.fs</Link>
+    </Compile>
     <Compile Include="$(FSharpSourcesRoot)\..\tests\service\Program.fs" Condition="'$(TargetFramework)' == 'netcoreapp3.0'">
       <Link>Program.fs</Link>
     </Compile>
@@ -71,10 +75,10 @@
   <ItemGroup>
     <PackageReference Include="FSharp.Core" Version="$(FcsFSharpCorePkgVersion)" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.6.0" />
-    <PackageReference Include="NUnit" Version="3.9.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.9.0" />
-    <PackageReference Include="NunitXml.TestLogger" Version="2.1.36" />
     <PackageReference Include="Dotnet.ProjInfo" Version="0.20.0" />
+    <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
+    <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
+    <PackageReference Include="NunitXml.TestLogger" Version="$(NunitXmlTestLoggerVersion)" />
     <ProjectReference Include="..\FSharp.Compiler.Service\FSharp.Compiler.Service.fsproj" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == '$(FcsTargetNetFxFramework)'">

fcs/build.fsx

@@ -73,9 +73,9 @@ Target.create "Test" (fun _ ->
     // This project file is used for the netcoreapp2.0 tests to work out reference sets
     runDotnet __SOURCE_DIRECTORY__ "build" "../tests/projects/Sample_NETCoreSDK_FSharp_Library_netstandard2_0/Sample_NETCoreSDK_FSharp_Library_netstandard2_0.fsproj -nodereuse:false -v n /restore /p:DisableCompilerRedirection=true"
 
-    // Now run the tests
-    let logFilePath = Path.Combine(__SOURCE_DIRECTORY__, "..", "artifacts", "TestResults", "Release", "FSharp.Compiler.Service.Test.xml")
-    runDotnet __SOURCE_DIRECTORY__ "test" (sprintf "FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj --no-restore --no-build -nodereuse:false -v n -c Release --test-adapter-path . --logger \"nunit;LogFilePath=%s\"" logFilePath)
+    // Now run the tests (different output files per TFM)
+    let logFilePath = Path.Combine(__SOURCE_DIRECTORY__, "..", "artifacts", "TestResults", "Release", "FSharp.Compiler.Service.Test.{framework}.xml")
+    runDotnet __SOURCE_DIRECTORY__ "test" (sprintf "FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj --no-restore --no-build -nodereuse:false -v n -c Release --logger \"nunit;LogFilePath=%s\"" logFilePath)
 )
 
 Target.create "NuGet" (fun _ ->

fcs/samples/FscExe/FscMain.fs

@@ -8,7 +8,7 @@ open System.IO
 open System.Reflection
 open System.Runtime.CompilerServices
 open FSharp.Compiler.SourceCodeServices
-open FSharp.Compiler.AbstractIL.IL // runningOnMono 
+open FSharp.Compiler.AbstractIL.Internal.Utils // runningOnMono
 open FSharp.Compiler.AbstractIL.Internal.Library
 open FSharp.Compiler.ErrorLogger
 

src/absil/bytes.fs

@@ -12,6 +12,25 @@ open FSharp.NativeInterop
 
 #nowarn "9"
 
+module Utils =
+    let runningOnMono =
+    #if ENABLE_MONO_SUPPORT
+        // Officially supported way to detect if we are running on Mono.
+        // See http://www.mono-project.com/FAQ:_Technical
+        // "How can I detect if am running in Mono?" section
+        try
+            System.Type.GetType ("Mono.Runtime") <> null
+        with _ ->
+            // Must be robust in the case that someone else has installed a handler into System.AppDomain.OnTypeResolveEvent
+            // that is not reliable.
+            // This is related to bug 5506--the issue is actually a bug in VSTypeResolutionService.EnsurePopulated which is
+            // called by OnTypeResolveEvent. The function throws a NullReferenceException. I'm working with that team to get
+            // their issue fixed but we need to be robust here anyway.
+            false
+    #else
+        false
+    #endif
+
 module internal Bytes = 
     let b0 n =  (n &&& 0xFF)
     let b1 n =  ((n >>> 8) &&& 0xFF)
@@ -296,65 +315,72 @@ type ByteMemory with
     member x.AsReadOnly() = ReadOnlyByteMemory x
 
     static member CreateMemoryMappedFile(bytes: ReadOnlyByteMemory) =
-        let length = int64 bytes.Length
-        let mmf = 
+        if Utils.runningOnMono
+        then
+            // mono's MemoryMappedFile implementation throws with null `mapName`, so we use byte arrays instead: https://github.com/mono/mono/issues/10245
+            ByteArrayMemory.FromArray (bytes.ToArray()) :> ByteMemory
+        else
+            let length = int64 bytes.Length
             let mmf =
-                MemoryMappedFile.CreateNew(
-                    null, 
-                    length, 
-                    MemoryMappedFileAccess.ReadWrite, 
-                    MemoryMappedFileOptions.None, 
-                    HandleInheritability.None)
-            use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite)
-            bytes.CopyTo stream
-            mmf
-
-        let accessor = mmf.CreateViewAccessor(0L, length, MemoryMappedFileAccess.ReadWrite)
-        RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, (mmf, accessor))
+                let mmf = MemoryMappedFile.CreateNew(null, length, MemoryMappedFileAccess.ReadWrite, MemoryMappedFileOptions.None, HandleInheritability.None)
+                use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite)
+                bytes.CopyTo stream
+                mmf
+
+            let accessor = mmf.CreateViewAccessor(0L, length, MemoryMappedFileAccess.ReadWrite)
+            RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, (mmf, accessor))
 
     static member FromFile(path, access, ?canShadowCopy: bool) =
         let canShadowCopy = defaultArg canShadowCopy false
 
-        let memoryMappedFileAccess =
-            match access with
-            | FileAccess.Read -> MemoryMappedFileAccess.Read
-            | FileAccess.Write -> MemoryMappedFileAccess.Write
-            | _ -> MemoryMappedFileAccess.ReadWrite
+        if Utils.runningOnMono
+        then
+            // mono's MemoryMappedFile implementation throws with null `mapName`, so we use byte arrays instead: https://github.com/mono/mono/issues/10245
+            let bytes = File.ReadAllBytes path
+            ByteArrayMemory.FromArray bytes
+        else
+            let memoryMappedFileAccess =
+                match access with
+                | FileAccess.Read -> MemoryMappedFileAccess.Read
+                | FileAccess.Write -> MemoryMappedFileAccess.Write
+                | _ -> MemoryMappedFileAccess.ReadWrite
 
-        let mmf, accessor, length = 
             let fileStream = File.Open(path, FileMode.Open, access, FileShare.Read)
+
             let length = fileStream.Length
-            let mmf = 
-                if canShadowCopy then
-                    let mmf = 
-                        MemoryMappedFile.CreateNew(
-                            null, 
-                            length, 
-                            MemoryMappedFileAccess.ReadWrite, 
-                            MemoryMappedFileOptions.None, 
-                            HandleInheritability.None)
-                    use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite)
-                    fileStream.CopyTo(stream)
-                    fileStream.Dispose()
-                    mmf
-                else
-                    MemoryMappedFile.CreateFromFile(
-                        fileStream, 
-                        null, 
-                        length, 
-                        memoryMappedFileAccess, 
-                        HandleInheritability.None, 
-                        leaveOpen=false)
-            mmf, mmf.CreateViewAccessor(0L, length, memoryMappedFileAccess), length
-
-        // Validate MMF with the access that was intended.
-        match access with
-        | FileAccess.Read when not accessor.CanRead -> invalidOp "Cannot read file"
-        | FileAccess.Write when not accessor.CanWrite -> invalidOp "Cannot write file"
-        | FileAccess.ReadWrite when not accessor.CanRead || not accessor.CanWrite -> invalidOp "Cannot read or write file"
-        | _ -> ()
-
-        RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, (mmf, accessor))
+
+            let mmf, accessor, length =
+                let mmf =
+                    if canShadowCopy then
+                        let mmf =
+                            MemoryMappedFile.CreateNew(
+                                null,
+                                length,
+                                MemoryMappedFileAccess.ReadWrite,
+                                MemoryMappedFileOptions.None,
+                                HandleInheritability.None)
+                        use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite)
+                        fileStream.CopyTo(stream)
+                        fileStream.Dispose()
+                        mmf
+                    else
+                        MemoryMappedFile.CreateFromFile(
+                            fileStream,
+                            null,
+                            length,
+                            memoryMappedFileAccess,
+                            HandleInheritability.None,
+                            leaveOpen=false)
+                mmf, mmf.CreateViewAccessor(0L, length, memoryMappedFileAccess), length
+
+            // Validate MMF with the access that was intended.
+            match access with
+            | FileAccess.Read when not accessor.CanRead -> invalidOp "Cannot read file"
+            | FileAccess.Write when not accessor.CanWrite -> invalidOp "Cannot write file"
+            | FileAccess.ReadWrite when not accessor.CanRead || not accessor.CanWrite -> invalidOp "Cannot read or write file"
+            | _ -> ()
+
+            RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, (mmf, accessor))
 
     static member FromUnsafePointer(addr, length, holder: obj) = 
         RawByteMemory(NativePtr.ofNativeInt addr, length, holder) :> ByteMemory

src/absil/bytes.fsi

@@ -5,10 +5,11 @@ namespace FSharp.Compiler.AbstractIL.Internal
 
 open System.IO
 open Internal.Utilities
-
 open FSharp.Compiler.AbstractIL 
 open FSharp.Compiler.AbstractIL.Internal 
 
+module Utils =
+    val runningOnMono: bool
 
 module internal Bytes = 
     /// returned int will be 0 <= x <= 255

src/absil/il.fs

@@ -25,24 +25,6 @@ open Internal.Utilities
 
 let logging = false
 
-let runningOnMono =
-#if ENABLE_MONO_SUPPORT
-// Officially supported way to detect if we are running on Mono.
-// See http://www.mono-project.com/FAQ:_Technical
-// "How can I detect if am running in Mono?" section
-    try
-        System.Type.GetType ("Mono.Runtime") <> null
-    with e->
-        // Must be robust in the case that someone else has installed a handler into System.AppDomain.OnTypeResolveEvent
-        // that is not reliable.
-        // This is related to bug 5506--the issue is actually a bug in VSTypeResolutionService.EnsurePopulated which is
-        // called by OnTypeResolveEvent. The function throws a NullReferenceException. I'm working with that team to get
-        // their issue fixed but we need to be robust here anyway.
-        false
-#else
-    false
-#endif
-
 let _ = if logging then dprintn "* warning: Il.logging is on"
 
 let int_order = LanguagePrimitives.FastGenericComparer<int>

src/absil/il.fsi

@@ -2012,8 +2012,6 @@ type ILPropertyRef =
      member Name: string
      interface System.IComparable
 
-val runningOnMono: bool
-
 type ILReferences = 
     { AssemblyReferences: ILAssemblyRef list 
       ModuleReferences: ILModuleRef list }

src/absil/ilread.fs

@@ -20,12 +20,13 @@ open System.Text
 open Internal.Utilities
 open Internal.Utilities.Collections
 open FSharp.NativeInterop
-open FSharp.Compiler.AbstractIL.Internal 
-open FSharp.Compiler.AbstractIL.Internal.Support
 open FSharp.Compiler.AbstractIL.Diagnostics 
+open FSharp.Compiler.AbstractIL.IL
+open FSharp.Compiler.AbstractIL.Internal
 open FSharp.Compiler.AbstractIL.Internal.BinaryConstants 
-open FSharp.Compiler.AbstractIL.IL  
 open FSharp.Compiler.AbstractIL.Internal.Library
+open FSharp.Compiler.AbstractIL.Internal.Support
+open FSharp.Compiler.AbstractIL.Internal.Utils
 open FSharp.Compiler.ErrorLogger
 open FSharp.Compiler.Range
 open System.Reflection

src/absil/ilreflect.fs

@@ -17,6 +17,7 @@ open System.Collections.Generic
 open FSharp.Compiler.AbstractIL
 open FSharp.Compiler.AbstractIL.Internal
 open FSharp.Compiler.AbstractIL.Internal.Library
+open FSharp.Compiler.AbstractIL.Internal.Utils
 open FSharp.Compiler.AbstractIL.Diagnostics 
 open FSharp.Compiler.AbstractIL.IL
 open FSharp.Compiler.ErrorLogger