diff --git a/CMakeLists.txt b/CMakeLists.txt index 197ea538bc25a..f554d821f052e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1294,9 +1294,9 @@ if(swift_build_android AND NOT "${SWIFT_HOST_VARIANT_SDK}" STREQUAL "ANDROID") if ("${SWIFT_ANDROID_NDK_PATH}" STREQUAL "") message(FATAL_ERROR "You must set SWIFT_ANDROID_NDK_PATH to cross-compile the Swift runtime for Android") endif() - if (NOT ("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Darwin" OR "${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Linux")) - message(FATAL_ERROR "A Darwin or Linux host is required to build the Swift runtime for Android") - endif() + #if (NOT ("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Darwin" OR "${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Linux")) + # message(FATAL_ERROR "A Darwin or Linux host is required to build the Swift runtime for Android") + #endif() if("${SWIFT_SDK_ANDROID_ARCHITECTURES}" STREQUAL "") set(SWIFT_SDK_ANDROID_ARCHITECTURES armv7;aarch64) diff --git a/cmake/caches/Runtime-Android-x86_64.cmake b/cmake/caches/Runtime-Android-x86_64.cmake index 8780313aaa3ca..86b36a453f3a1 100644 --- a/cmake/caches/Runtime-Android-x86_64.cmake +++ b/cmake/caches/Runtime-Android-x86_64.cmake @@ -1,4 +1,5 @@ +set(SWIFT_HOST_VARIANT Windows CACHE STRING "") set(SWIFT_HOST_VARIANT_SDK ANDROID CACHE STRING "") set(SWIFT_HOST_VARIANT_ARCH x86_64 CACHE STRING "") diff --git a/test/lit.cfg b/test/lit.cfg index cd6931963bc61..71249f918ae08 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -1891,9 +1891,11 @@ elif run_os == 'linux-androideabi' or run_os == 'linux-android': config.target_pic_opt = "-fPIC" ndk_platform_tuple = get_architecture_value(armv7="armeabi-v7a", - aarch64="arm64-v8a") + aarch64="arm64-v8a", + x86_64="x86_64") ndk_platform_triple = get_architecture_value(armv7="arm-linux-androideabi", - aarch64="aarch64-linux-android") + aarch64="aarch64-linux-android", + x86_64="x86_64-linux-android") if platform.system() == 'Linux': prebuilt_directory = 'linux-x86_64' elif platform.system() == 'Darwin': @@ -1944,7 +1946,7 @@ elif run_os == 'linux-androideabi' or run_os == 'linux-android': mcp_opt, config.swift_test_options, config.swift_frontend_test_options]) subst_target_swift_frontend_mock_sdk = config.target_swift_frontend subst_target_swift_frontend_mock_sdk_after = "" - config.target_run = make_path(config.swift_src_root, 'utils', 'android', 'adb_test_runner.py') + config.target_run = '"' + sys.executable + '" ' + make_path(config.swift_src_root, 'utils', 'android', 'adb_test_runner.py') # Use SDKROOT instead of -sdk because a couple tests set another -sdk and # sil-opt errors if passed -sdk multiple times. config.target_sil_opt = ' '.join([ diff --git a/utils/android/adb/commands.py b/utils/android/adb/commands.py index f7bf0680e79e8..fbe4e009618ad 100644 --- a/utils/android/adb/commands.py +++ b/utils/android/adb/commands.py @@ -26,6 +26,11 @@ ENV_PREFIX = 'ANDROID_CHILD_' +# Subprocess output might be bytes or str depending on OS and Python version +def _as_str(s): + return s if isinstance(s, str) else str(s, 'utf-8') + + def shell(args): """ Execute 'adb shell' with the given arguments. @@ -54,7 +59,7 @@ def push(local_paths, device_path): ['adb', 'push', '--sync'] + local_paths + [device_path], stderr=subprocess.STDOUT).strip() except subprocess.CalledProcessError as e: - if "unrecognized option '--sync'" in e.output: + if "unrecognized option '--sync'" in _as_str(e.output): return subprocess.check_output( ['adb', 'push'] + local_paths + [device_path], stderr=subprocess.STDOUT).strip() @@ -109,6 +114,7 @@ def execute_on_device(executable_path, executable_arguments): executable_name = os.path.basename(executable_path) executable = '{}/{}'.format(uuid_dir, executable_name) push(executable_path, executable) + shell(['chmod', '+x', executable]) child_environment = ['{}="{}"'.format(k.replace(ENV_PREFIX, '', 1), v) for (k, v) in os.environ.items() @@ -167,8 +173,8 @@ def execute_on_device(executable_path, executable_arguments): shell([executable_piped]) # Grab the results of running the executable on device. - stdout = shell(['cat', executable_stdout]) - exitcode = shell(['cat', executable_succeeded]) + stdout = _as_str(shell(['cat', executable_stdout])) + exitcode = _as_str(shell(['cat', executable_succeeded])) if not exitcode.startswith(succeeded_token): debug_command = '$ adb shell {}'.format(executable_with_args) print('Executable exited with a non-zero code on the Android device.\n' diff --git a/utils/android/adb_push_built_products/main.py b/utils/android/adb_push_built_products/main.py index d9977d2eeabf4..d1894ff8d5264 100644 --- a/utils/android/adb_push_built_products/main.py +++ b/utils/android/adb_push_built_products/main.py @@ -18,6 +18,7 @@ import argparse import glob import os +import sys import adb.commands @@ -46,7 +47,7 @@ def argument_parser(): '-a', '--destination-arch', help='The architecture of the host device. Used to determine the ' 'right library versions to send to the device.', - choices=['armv7', 'aarch64'], + choices=['armv7', 'aarch64', 'x86_64'], default='armv7') return parser @@ -56,6 +57,29 @@ def _push(sources, destination): print(adb.commands.push(sources, destination)) +def _find_libcpp(ndk_path, target_arch): + ndk_version = 26 + # TODO: Add argument and keep support? + if ndk_version < 22: + return os.path.join(ndk_path, + 'sources', + 'cxx-stl', + 'llvm-libc++', + 'libs', + { + 'armv7': 'armeabi-v7a', + 'aarch64': 'arm64-v8a', + 'x86_64': 'x86_64' + }[target_arch], + 'libc++_shared.so') + # TODO: Detect host arch + host_triple = 'windows-x86_64' if sys.platform == 'win32' else 'linux-x86_64' + target_triple = target_arch + '-linux-android' + + # TODO: Support just-built libc++? + return os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', host_triple, 'sysroot', 'usr', 'lib', target_triple, 'libc++_shared.so') + + def main(): """ The main entry point for adb_push_built_products. @@ -67,6 +91,7 @@ def main(): args = parser.parse_args() for path in args.paths: + adb.commands.shell(['mkdir', '-p', args.destination]) if os.path.isdir(path): full_paths = [ os.path.join(path, basename) @@ -76,16 +101,7 @@ def main(): _push(path, args.destination) if args.ndk: - libcpp = os.path.join(args.ndk, - 'sources', - 'cxx-stl', - 'llvm-libc++', - 'libs', - { - 'armv7': 'armeabi-v7a', - 'aarch64': 'arm64-v8a', - }[args.destination_arch], - 'libc++_shared.so') + libcpp = _find_libcpp(args.ndk, args.destination_arch) _push(libcpp, args.destination) return 0 diff --git a/utils/build-windows-toolchain.bat b/utils/build-windows-toolchain.bat index 131c283eddc1e..290c26772d88a 100644 --- a/utils/build-windows-toolchain.bat +++ b/utils/build-windows-toolchain.bat @@ -60,7 +60,7 @@ set TMPDIR=%BuildRoot%\tmp set NINJA_STATUS=[%%f/%%t][%%p][%%es] :: Build the -Test argument, if any, by subtracting skipped tests -set TestArg=-Test lld,lldb,swift,dispatch,foundation,xctest,swift-format,sourcekit-lsp, +set TestArg=-Test android, for %%I in (%SKIP_TESTS%) do (call set TestArg=%%TestArg:%%I,=%%) if "%TestArg:~-1%"=="," (set TestArg=%TestArg:~0,-1%) else (set TestArg= ) @@ -77,6 +77,8 @@ powershell.exe -ExecutionPolicy RemoteSigned -File %~dp0build.ps1 ^ -ImageRoot %BuildRoot% ^ %SkipPackagingArg% ^ %TestArg% ^ + -AndroidSDKs x86_64 ^ + -WindowsSDKs X64 ^ -Stage %PackageRoot% ^ -Summary || (exit /b 1) diff --git a/utils/build.ps1 b/utils/build.ps1 index 90838ae3d0f3f..6131e8fcc1b3f 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -204,7 +204,7 @@ if ($AndroidSDKs.Length -gt 0) { if ($Test -contains "*") { # Explicitly don't include llbuild yet since tests are known to fail on Windows - $Test = @("lld", "lldb", "swift", "dispatch", "foundation", "xctest", "swift-format", "sourcekit-lsp") + $Test = @("lld", "lldb", "swift", "dispatch", "foundation", "xctest", "swift-format", "sourcekit-lsp", "android") } # Architecture definitions @@ -858,6 +858,53 @@ function Fetch-Dependencies { DownloadAndVerify $NDKURL "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip" $NDKHash Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" -CreateExtractPath $false + + # FIXME: Both Java and Android emulator must be available in the environment. + # This is a terrible workaround. It's a waste of time and resources. + $SDKDir = "$BinaryCache\android-sdk" + if ($Test -and -not(Test-Path "$SDKDir\licenses")) { + # Download Java Runtime + switch ($BuildArchName) { + "AMD64" { + $JavaURL = "https://aka.ms/download-jdk/microsoft-jdk-17.0.14-windows-x64.zip" + $JavaHash = "3619082f4a667f52c97cce983364a517993f798ae248c411765becfd9705767f" + } + "ARM64" { + $JavaURL = "https://aka.ms/download-jdk/microsoft-jdk-17.0.14-windows-aarch64.zip" + $JavaHash = "2a12c7b3d46712de9671f5f011a3cae9ee53d5ff3b0604136ee079a906146448" + } + default { throw "Unsupported processor architecture" } + } + DownloadAndVerify $JavaURL "$BinaryCache\microsoft-jdk.zip" $JavaHash + Extract-ZipFile -ZipFileName "microsoft-jdk.zip" -BinaryCache $BinaryCache -ExtractPath "android-sdk-jdk" + + # Download cmdline-tools + $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" + $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" + DownloadAndVerify $CLToolsURL "$BinaryCache\android-cmdline-tools.zip" $CLToolsHash + Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-sdk-cmdline-tools" + + # Accept licenses + New-Item -Type Directory -Path "$SDKDir\licenses" -ErrorAction Ignore | Out-Null + Set-Content -Path "$SDKDir\licenses\android-sdk-license" -Value "24333f8a63b6825ea9c5514f83c2829b004d1fee" + Set-Content -Path "$SDKDir\licenses\android-sdk-preview-license" -Value "84831b9409646a918e30573bab4c9c91346d8abd" + + # Install packages and create test device + Isolate-EnvVars { + $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" + $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" + + # Let cmdline-tools install itself. This is idiomatic for installing the Android SDK. + Invoke-Program "$BinaryCache\android-sdk-cmdline-tools\cmdline-tools\bin\sdkmanager.bat" -OutNull "--sdk_root=$SDKDir" '"cmdline-tools;latest"' '--channel=3' + $AndroidSdkMgr = "$SDKDir\cmdline-tools\latest\bin\sdkmanager.bat" + + # TODO: Install all platforms that we want to test on. + foreach ($Package in @('"system-images;android-29;default;x86_64"', '"platforms;android-29"', '"platform-tools"')) { + Write-Host "$AndroidSdkMgr $Package" + Invoke-Program -OutNull $AndroidSdkMgr $Package + } + } + } } if ($IncludeDS2) { @@ -909,6 +956,72 @@ function Fetch-Dependencies { } } +$AndroidEmulatorPid = $null +$AndroidEmulatorArchName = $null + +function AndroidEmulator-CreateDevice($ArchName) { + $DeviceName = "swift-test-device-$ArchName" + $Packages = "system-images;android-29;default;$ArchName" + $AvdTool = "$BinaryCache\android-sdk\cmdline-tools\latest\bin\avdmanager.bat" + Isolate-EnvVars { + $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" + $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" + $Output = & $AvdTool list avd + if ($Output -match $DeviceName) { + Write-Host "Found Android virtual device for arch $ArchName" + } else { + Write-Host "Create Android virtual device for arch $ArchName" + "no" | & $AvdTool create avd --force --name $DeviceName --package $Packages + } + } + return $DeviceName +} + +function AndroidEmulator-Run($ArchName) { + if ($AndroidEmulatorArchName -ne $ArchName) { + AndroidEmulator-TearDown + } + if ($AndroidEmulatorPid -eq $null) { + Write-Host "Start Android emulator for arch $ArchName" + $Device = (AndroidEmulator-CreateDevice $ArchName) + $EmuTool = "$BinaryCache\android-sdk\emulator\emulator.exe" + Isolate-EnvVars { + $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" + $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" + $Process = Start-Process -PassThru $EmuTool "@$Device" ` + -RedirectStandardOutput "$BinaryCache\android-sdk\.temp\emulator.out" ` + -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" + $global:AndroidEmulatorPid = $Process.Id + $global:AndroidEmulatorArchName = $ArchName + Write-Host "Waiting while Android emulator boots in process $AndroidEmulatorPid" + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + Invoke-Program $adb "wait-for-device" + } + } +} + +function AndroidEmulator-TearDown() { + if ($AndroidEmulatorPid -ne $null) { + if (Get-Process -Id $AndroidEmulatorPid -ErrorAction SilentlyContinue) { + Write-Host "Tear down Android emulator for arch $AndroidEmulatorArchName" + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + & $adb emu kill | Out-Null + & $adb kill-server | Out-Null + + try { + Write-Host "Waiting for process $AndroidEmulatorPid to exit" + Wait-Process -Id $AndroidEmulatorPid -Timeout 1 + } catch { + Write-Host "Process $AndroidEmulatorPid failed to exit. Shutting it down." + Stop-Process -Force -Id $AndroidEmulatorPid + } + + $global:AndroidEmulatorPid = $null + $global:AndroidEmulatorArchName = $null + } + } +} + function Get-PinnedToolchainTool() { if (Test-Path "$BinaryCache\toolchains\${PinnedToolchain}\LocalApp\Programs\Swift\Toolchains\$(Get-PinnedToolchainVersion)+Asserts\usr\bin") { return "$BinaryCache\toolchains\${PinnedToolchain}\LocalApp\Programs\Swift\Toolchains\$(Get-PinnedToolchainVersion)+Asserts\usr\bin" @@ -1997,17 +2110,29 @@ function Build-CURL([Platform]$Platform, $Arch) { }) } -function Build-Runtime([Platform]$Platform, $Arch) { +function Build-Runtime([Platform]$Platform, $Arch, [switch]$Test = $false) { $PlatformDefines = @{} if ($Platform -eq "Android") { $PlatformDefines += @{ LLVM_ENABLE_LIBCXX = "YES"; SWIFT_USE_LINKER = "lld"; - SWIFT_INCLUDE_TESTS = "NO"; - SWIFT_INCLUDE_TEST_BINARIES = "NO"; } } + if ($Test -and $Platform -eq "Android") { + $Targets = @("check-swift-all-only_non_executable-android-x86_64", "install") + #$Targets = @("upload-stdlib-android-x86_64", "check-swift-android-x86_64", "install") + #AndroidEmulator-Run $Arch.LLVMName + $PlatformDefines += @{ + SWIFT_INCLUDE_TESTS = "YES"; + SWIFT_BUILD_SDK_OVERLAY = "YES"; + SWIFT_BUILD_STDLIB = "YES"; + SWIFT_BUILD_TEST_SUPPORT_MODULES = "YES"; + SWIFT_ANDROID_DEPLOY_DEVICE_PATH = "/data/local/tmp" + } + } else { + $Targets = @("install") + } Isolate-EnvVars { $env:Path = "$(Get-CMarkBinaryCache $Arch)\src;$(Get-PinnedToolchainRuntime);${env:Path}" @@ -2018,6 +2143,7 @@ function Build-Runtime([Platform]$Platform, $Arch) { Get-HostProjectBinaryCache Compilers } + $NativeToolsPath = Join-Path -Path $CompilersBinaryCache -ChildPath "bin" Build-CMakeProject ` -Src $SourceCache\swift ` -Bin (Get-TargetProjectBinaryCache $Arch Runtime) ` @@ -2026,6 +2152,7 @@ function Build-Runtime([Platform]$Platform, $Arch) { -Platform $Platform ` -CacheScript $SourceCache\swift\cmake\caches\Runtime-$Platform-$($Arch.LLVMName).cmake ` -UseBuiltCompilers C,CXX,Swift ` + -BuildTargets $Targets ` -Defines ($PlatformDefines + @{ CMAKE_Swift_COMPILER_TARGET = (Get-ModuleTriple $Arch); CMAKE_Swift_COMPILER_WORKS = "YES"; @@ -2039,7 +2166,9 @@ function Build-Runtime([Platform]$Platform, $Arch) { SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING = "YES"; SWIFT_ENABLE_SYNCHRONIZATION = "YES"; SWIFT_ENABLE_VOLATILE = "YES"; - SWIFT_NATIVE_SWIFT_TOOLS_PATH = (Join-Path -Path $CompilersBinaryCache -ChildPath "bin"); + SWIFT_NATIVE_CLANG_TOOLS_PATH = $NativeToolsPath; + SWIFT_NATIVE_LLVM_TOOLS_PATH = $NativeToolsPath; + SWIFT_NATIVE_SWIFT_TOOLS_PATH = $NativeToolsPath; SWIFT_PATH_TO_LIBDISPATCH_SOURCE = "$SourceCache\swift-corelibs-libdispatch"; SWIFT_PATH_TO_STRING_PROCESSING_SOURCE = "$SourceCache\swift-experimental-string-processing"; CMAKE_SHARED_LINKER_FLAGS = if ($Platform -eq "Windows") { @("/INCREMENTAL:NO", "/OPT:REF", "/OPT:ICF") } else { @() }; @@ -3037,6 +3166,7 @@ if (-not $SkipBuild) { Invoke-BuildStep Write-PlatformInfoPlist $Arch } + $TestAndroid = $Test -contains "android" foreach ($Arch in $AndroidSDKArchs) { if ($IncludeDS2) { Invoke-BuildStep Build-DS2 Android $Arch @@ -3046,8 +3176,17 @@ if (-not $SkipBuild) { Invoke-BuildStep Build-CURL Android $Arch Invoke-BuildStep Build-LLVM Android $Arch + # FIXME: We need a separate step Test-Runtime, so -SkipBuild won't skip it. + # However, this will require a reconfiguration of the binary cache which + # might cause more trouble. Let's get it to wrok like this first. + #if ($Test -contains "android" -and $Arch -eq $AndroidX64) { + if ($Arch -eq $AndroidX64) { + Invoke-BuildStep Build-Runtime Android $Arch -Test + } else { + Invoke-BuildStep Build-Runtime Android $Arch + } + # Build platform: SDK, Redist and XCTest - Invoke-BuildStep Build-Runtime Android $Arch Invoke-BuildStep Build-Dispatch Android $Arch Invoke-BuildStep Build-Foundation Android $Arch Invoke-BuildStep Build-Sanitizers Android $Arch @@ -3175,6 +3314,8 @@ if (-not $IsCrossCompiling) { exit 1 } finally { + AndroidEmulator-TearDown + if ($Summary) { $TimingData | Select-Object Platform,Arch,Checkout,"Elapsed Time" | Sort-Object -Descending -Property "Elapsed Time" | Format-Table -AutoSize }