[windows][toolchain] Run non-executable Swift Runtime tests for Android (#79185)

We cross-compile the Swift runtime libs for the Android SDKs in the Windows toolchain. This patch adds a build step that runs non-executable tests for them.

Author: weliveindetail

test/lit.cfg

@@ -239,7 +239,9 @@ elif platform.system() == 'Darwin':
 config.substitutions.append( ('%lto_flags', lto_flags) )
 config.substitutions.append( ('%use_just_built_liblto', use_just_built_liblto) )
 config.substitutions.append( ('%llvm_libs_dir', llvm_libs_dir) )
-config.substitutions.append( ('%llvm_plugin_ext', llvm_plugin_ext) )
+
+if llvm_plugin_ext is not None:
+    config.substitutions.append( ('%llvm_plugin_ext', llvm_plugin_ext) )
 
 # Allow tests to restore the original environment if they need to.
 config.substitutions.append( ('%original_path_env', config.environment['PATH']) )
@@ -395,28 +397,6 @@ lit_config.note('Using resource dir: ' + test_resource_dir)
 test_clang_resource_dir = lit.util.executeCommand([config.clang, '-print-resource-dir'])[0].rstrip()
 lit_config.note('Using Clang resource dir: ' + test_clang_resource_dir)
 
-config.swift_system_overlay_opt = ""
-config.clang_system_overlay_opt = ""
-config.windows_vfs_overlay_opt = ""
-if kIsWindows:
-    # ucrt.modulemap currently requires -fbuiltin-headers-in-system-modules. -strict-implicit-module-context
-    # is necessary for -Xcc arguments to be passed through ModuleInterfaceLoader.
-    config.swift_system_overlay_opt = "-vfsoverlay {} -strict-implicit-module-context -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
-        os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
-    )
-    # this variant is for extract-symbolgraph which doesn't accept all the same arugments as swiftc
-    # so the extra ceremony lets us pass in the relevant pieces needed for Windows SDK access.
-    config.windows_vfs_overlay_opt = "-Xcc -vfsoverlay -Xcc {} -Xcc -strict-implicit-module-context -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
-        os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
-    )
-    config.clang_system_overlay_opt = "-Xcc -ivfsoverlay -Xcc {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
-        os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
-    )
-config.substitutions.append( ('%windows_vfs_overlay_opt', config.windows_vfs_overlay_opt) )
-stdlib_resource_dir_opt = config.resource_dir_opt
-sourcekitd_framework_dir = config.swift_lib_dir
-config.substitutions.append( ('%test-resource-dir', test_resource_dir) )
-
 # Parse the variant triple.
 # FIXME: We ought to parse 'run_environment' separately from 'run_os'.
 (run_cpu, run_vendor, run_os, run_vers) = re.match('([^-]+)-([^-]+)-([^0-9]+)(.*)', config.variant_triple).groups()
@@ -479,6 +459,28 @@ if test_sdk_overlay_dir is not None:
     lit_config.note('Using SDK overlay dir: ' + test_sdk_overlay_dir)
     config.resource_dir_opt += (" %s" % sdk_overlay_dir_opt)
 
+config.swift_system_overlay_opt = ""
+config.clang_system_overlay_opt = ""
+config.windows_vfs_overlay_opt = ""
+if kIsWindows and run_os in ['windows-msvc']:
+    # ucrt.modulemap currently requires -fbuiltin-headers-in-system-modules. -strict-implicit-module-context
+    # is necessary for -Xcc arguments to be passed through ModuleInterfaceLoader.
+    config.swift_system_overlay_opt = "-vfsoverlay {} -strict-implicit-module-context -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
+        os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
+    )
+    # this variant is for extract-symbolgraph which doesn't accept all the same arugments as swiftc
+    # so the extra ceremony lets us pass in the relevant pieces needed for Windows SDK access.
+    config.windows_vfs_overlay_opt = "-Xcc -vfsoverlay -Xcc {} -Xcc -strict-implicit-module-context -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
+        os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
+    )
+    config.clang_system_overlay_opt = "-Xcc -ivfsoverlay -Xcc {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
+        os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
+    )
+config.substitutions.append( ('%windows_vfs_overlay_opt', config.windows_vfs_overlay_opt) )
+stdlib_resource_dir_opt = config.resource_dir_opt
+sourcekitd_framework_dir = config.swift_lib_dir
+config.substitutions.append( ('%test-resource-dir', test_resource_dir) )
+
 # Default to Swift 4 for now.
 # Note that this accepts both `--param swift-version` (like the compiler flag)
 # and `--param swift_version` (like a lit configuration parameter).
@@ -1944,6 +1946,7 @@ elif run_os == 'linux-androideabi' or run_os == 'linux-android':
     config.target_swift_symbolgraph_extract = ' '.join([
         config.swift_symbolgraph_extract,
         '-target', config.variant_triple,
+        '-sdk', config.variant_sdk, config.resource_dir_opt,
         mcp_opt])
     config.target_swift_synthesize_interface = ' '.join([
         config.swift_synthesize_interface,
@@ -2747,6 +2750,7 @@ config.substitutions.insert(0, ('%check-c-header-in-clang',
 # Verifies that a C++ file can be compiled without warnings except for some exceptions.
 config.substitutions.insert(0, ('%check-cxx-header-in-clang',
   '%clangxx -fsyntax-only -x c++-header ' + extraCheckWarningFlags +
+  ' --sysroot %r ' % config.variant_sdk +
   # Use standard header/framework search paths.
   '-F %%clang-importer-sdk-path/frameworks '
   '-I %%clang-include-dir %%cxx-stdlib-include '

test/refactoring/lit.local.cfg

@@ -1,2 +1,2 @@
-config.substitutions.append((r'%refactor-check-compiles', f'{config.python} {config.refactor_check_compiles} -swift-refactor {config.swift_refactor} -swift-frontend {config.swift_frontend} -temp-dir %t {config.resource_dir_opt}'))
-config.substitutions.append((r'%refactor', f'{config.swift_refactor} {config.resource_dir_opt}'))
+config.substitutions.append((r'%refactor-check-compiles', f'{config.python} {config.refactor_check_compiles} -swift-refactor {config.swift_refactor} -swift-frontend {config.swift_frontend} -temp-dir %t {config.resource_dir_opt} -target {config.variant_triple}'))
+config.substitutions.append((r'%refactor', f'{config.swift_refactor} {config.resource_dir_opt} -target {config.variant_triple}'))

utils/build.ps1

@@ -1642,6 +1642,24 @@ function Write-PList {
       -OutFile $Path
 }
 
+function Load-LitTestOverrides($Filename) {
+  function Select-LitTestOverrides($Prefix) {
+    $MatchingLines = Get-Content $Filename | Select-String -Pattern "`^${Prefix}.*$"
+    return $MatchingLines | ForEach-Object { ($_ -replace $Prefix,"").Trim() }
+  }
+
+  $TestsToXFail = Select-LitTestOverrides "xfail"
+  Write-Host "TestsToXFail=$TestsToXFail"
+  if ($TestsToXFail -and $TestsToXFail.Length -ne 0) {
+    $env:LIT_XFAIL = $TestsToXFail -join ";"
+  }
+  $TestsToSkip = Select-LitTestOverrides "skip"
+  Write-Host "TestsToSkip=$TestsToSkip"
+  if ($TestsToSkip -and $TestsToSkip.Length -gt 0) {
+    $env:LIT_FILTER_OUT = "($($TestsToSkip -join '|'))"
+  }
+}
+
 function Build-Compilers() {
   [CmdletBinding(PositionalBinding = $false)]
   param
@@ -1681,19 +1699,8 @@ function Build-Compilers() {
       if ($TestLLDB) {
         $Targets += @("check-lldb")
 
-        function Select-LitTestOverrides {
-          param([string] $TestStatus)
-
-          $MatchingLines=(Get-Content $PSScriptRoot/windows-llvm-lit-test-overrides.txt | Select-String -Pattern "`^${TestStatus}.*$")
-          $TestNames=$MatchingLines | ForEach-Object { ($_ -replace $TestStatus,"").Trim() }
-          return $TestNames
-        }
-
-        # Override some test results with llvm-lit.
-        $TestsToXFail=Select-LitTestOverrides "xfail"
-        $TestsToSkip=Select-LitTestOverrides "skip"
-        $env:LIT_XFAIL=$TestsToXFail -join ";"
-        $env:LIT_FILTER_OUT="($($TestsToSkip -join '|'))"
+        # Override test filter for known issues in downstream LLDB
+        Load-LitTestOverrides $PSScriptRoot/windows-llvm-lit-test-overrides.txt
 
         # Transitive dependency of _lldb.pyd
         $RuntimeBinaryCache = Get-TargetProjectBinaryCache $Arch Runtime
@@ -2143,8 +2150,6 @@ function Build-Runtime([Platform]$Platform, $Arch) {
     $PlatformDefines += @{
       LLVM_ENABLE_LIBCXX = "YES";
       SWIFT_USE_LINKER = "lld";
-      SWIFT_INCLUDE_TESTS = "NO";
-      SWIFT_INCLUDE_TEST_BINARIES = "NO";
       # Clang[<18] doesn't provide the _Builtin_float module.
       SWIFT_BUILD_CLANG_OVERLAYS_SKIP_BUILTIN_FLOAT = "YES";
     }
@@ -2188,6 +2193,40 @@ function Build-Runtime([Platform]$Platform, $Arch) {
   }
 }
 
+function Test-Runtime([Platform]$Platform, $Arch) {
+  if ($IsCrossCompiling) {
+    throw "Swift runtime tests are not supported when cross-compiling"
+  }
+  if (-not (Test-Path (Get-TargetProjectBinaryCache $Arch Runtime))) {
+    throw "Swift runtime tests are supposed to reconfigure the existing build"
+  }
+  $CompilersBinaryCache = Get-HostProjectBinaryCache Compilers
+  if (-not (Test-Path "$CompilersBinaryCache\bin\FileCheck.exe")) {
+    # These will exist if we test any of llvm/clang/lldb/lld/swift as well
+    throw "LIT test utilities not found in $CompilersBinaryCache\bin"
+  }
+
+  Invoke-IsolatingEnvVars {
+    # Filter known issues when testing on Windows
+    Load-LitTestOverrides $PSScriptRoot/windows-swift-android-lit-test-overrides.txt
+    $env:Path = "$(Get-CMarkBinaryCache $Arch)\src;$(Get-PinnedToolchainRuntime);${env:Path};$UnixToolsBinDir"
+    Build-CMakeProject `
+      -Src $SourceCache\swift `
+      -Bin (Get-TargetProjectBinaryCache $Arch Runtime) `
+      -Arch $Arch `
+      -Platform $Platform `
+      -UseBuiltCompilers C,CXX,Swift `
+      -BuildTargets check-swift-validation-only_non_executable `
+      -Defines @{
+        SWIFT_INCLUDE_TESTS = "YES";
+        SWIFT_INCLUDE_TEST_BINARIES = "YES";
+        SWIFT_BUILD_TEST_SUPPORT_MODULES = "YES";
+        SWIFT_NATIVE_LLVM_TOOLS_PATH = Join-Path -Path $CompilersBinaryCache -ChildPath "bin";
+        LLVM_LIT_ARGS = "-vv";
+      }
+  }
+}
+
 function Build-ExperimentalRuntime {
   [CmdletBinding(PositionalBinding = $false)]
   param
@@ -3327,6 +3366,12 @@ if (-not $IsCrossCompiling) {
   if ($Test -contains "swiftpm") { Test-PackageManager $HostArch }
   if ($Test -contains "swift-format") { Test-Format }
   if ($Test -contains "sourcekit-lsp") { Test-SourceKitLSP }
+
+  if ($Test -contains "swift") {
+    foreach ($Arch in $AndroidSDKArchs) {
+      Test-Runtime Android $Arch
+    }
+  }
 }
 
 # Custom exception printing for more detailed exception information