[SYCL] Remove plugin interface (#14145)

Co-authored-by: Callum Fare <[email protected]>
Co-authored-by: omarahmed1111 <[email protected]>
Co-authored-by: Kenneth Benzie (Benie) <[email protected]>
Co-authored-by: Martin Morrison-Grant <[email protected]>
Co-authored-by: omarahmed1111 <[email protected]>

Author: 5 people

.github/workflows/sycl-linux-run-tests.yml

@@ -261,7 +261,7 @@ jobs:
         echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
     - run: which clang++ sycl-ls
     - run: sycl-ls --verbose
-    - run: SYCL_PI_TRACE=-1 sycl-ls
+    - run: SYCL_UR_TRACE=1 sycl-ls
     - run: |
           if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then
             cat /usr/local/lib/igc/IGCTAG.txt

sycl/CMakeLists.txt

@@ -22,10 +22,27 @@ if (NOT DEFINED SYCL_ENABLE_PLUGINS)
   set(SYCL_ENABLE_PLUGINS "opencl;level_zero")
 endif()
 
+# Option to enable online kernel fusion via a JIT compiler
+option(SYCL_ENABLE_KERNEL_FUSION "Enable kernel fusion via JIT compiler" ON)
+if(SYCL_ENABLE_KERNEL_FUSION AND WIN32)
+  message(WARNING "Kernel fusion not yet supported on Windows")
+  set(SYCL_ENABLE_KERNEL_FUSION OFF CACHE
+      BOOL "Kernel fusion not yet supported on Windows" FORCE)
+endif()
+
+if (NOT XPTI_INCLUDES)
+  set(XPTI_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/../xpti/include)
+endif()
+
+if (NOT XPTI_PROXY_SRC)
+  set(XPTI_PROXY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../xpti/src/xpti_proxy.cpp)
+endif()
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 include(AddSYCLExecutable)
 include(AddSYCL)
 include(SYCLUtils)
+include(FetchUnifiedRuntime)
 
 # The change in SYCL_MAJOR_VERSION must be accompanied with the same update in
 # llvm/clang/lib/Driver/CMakeLists.txt.
@@ -144,14 +161,6 @@ install(DIRECTORY ${OpenCL_INCLUDE_DIR}/CL
         DESTINATION ${SYCL_INCLUDE_DIR}/sycl
         COMPONENT OpenCL-Headers)
 
-# Option to enable online kernel fusion via a JIT compiler
-option(SYCL_ENABLE_KERNEL_FUSION "Enable kernel fusion via JIT compiler" ON)
-if(SYCL_ENABLE_KERNEL_FUSION AND WIN32)
-  message(WARNING "Kernel fusion not yet supported on Windows")
-  set(SYCL_ENABLE_KERNEL_FUSION OFF CACHE 
-      BOOL "Kernel fusion not yet supported on Windows" FORCE)
-endif()
-
 # Option for enabling building the SYCL major release preview library.
 option(SYCL_ENABLE_MAJOR_RELEASE_PREVIEW_LIB "Enable build of the SYCL major release preview library" ON)
 
@@ -231,6 +240,8 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E copy_directory ${sycl_inc_dir}/std ${SYCL_INCLUDE_BUILD_DIR}/std
   COMMAND ${CMAKE_COMMAND} -E copy_directory ${sycl_inc_dir}/syclcompat ${SYCL_INCLUDE_BUILD_DIR}/syclcompat
   COMMAND ${CMAKE_COMMAND} -E copy ${sycl_inc_dir}/syclcompat.hpp ${SYCL_INCLUDE_BUILD_DIR}/syclcompat.hpp
+  COMMAND ${CMAKE_COMMAND} -E copy ${UNIFIED_RUNTIME_INCLUDE_DIR}/ur_api.h ${SYCL_INCLUDE_BUILD_DIR}/sycl
+  COMMAND ${CMAKE_COMMAND} -E copy ${UNIFIED_RUNTIME_INCLUDE_DIR}/ur_print.hpp ${SYCL_INCLUDE_BUILD_DIR}/sycl
   COMMENT "Copying SYCL headers ...")
 
 # Copy SYCL headers from source to install directory
@@ -240,6 +251,10 @@ install(DIRECTORY "${sycl_inc_dir}/std" DESTINATION ${SYCL_INCLUDE_DIR} COMPONEN
 install(DIRECTORY ${BOOST_MP11_DESTINATION_DIR} DESTINATION ${SYCL_INCLUDE_DIR}/sycl/detail COMPONENT boost_mp11-headers)
 install(DIRECTORY "${sycl_inc_dir}/syclcompat" DESTINATION ${SYCL_INCLUDE_DIR} COMPONENT sycl-headers)
 install(FILES "${sycl_inc_dir}/syclcompat.hpp" DESTINATION ${SYCL_INCLUDE_DIR} COMPONENT sycl-headers)
+install(FILES "${UNIFIED_RUNTIME_INCLUDE_DIR}/ur_api.h" DESTINATION ${SYCL_INCLUDE_DIR}/sycl
+  COMPONENT sycl-headers)
+install(FILES "${UNIFIED_RUNTIME_INCLUDE_DIR}/ur_print.hpp" DESTINATION ${SYCL_INCLUDE_DIR}/sycl
+  COMPONENT sycl-headers)
 
 if (WIN32)
   set(SYCL_RT_LIBS sycl${SYCL_MAJOR_VERSION})
@@ -354,7 +369,7 @@ add_custom_target( sycl-toolchain ALL
 )
 
 if (WIN32)
-  add_dependencies(sycl-toolchain pi_win_proxy_loader)
+  add_dependencies(sycl-toolchain ur_win_proxy_loader)
 endif()
 
 # Enable new IN_LIST operator.
@@ -384,13 +399,10 @@ if(NOT "${SYCL_BUILD_PI_HIP_PLATFORM}" STREQUAL "")
   string(TOUPPER ${SYCL_BUILD_PI_HIP_PLATFORM} SYCL_BUILD_PI_HIP_PLATFORM)
 endif()
 
-# Plugin Library
-add_subdirectory( plugins )
-
 add_subdirectory(tools)
 
 if (WIN32)
-  add_subdirectory(pi_win_proxy_loader)
+  add_subdirectory(ur_win_proxy_loader)
 endif()
 
 if(SYCL_INCLUDE_TESTS)
@@ -441,7 +453,7 @@ set( SYCL_TOOLCHAIN_DEPLOY_COMPONENTS
 )
 
 if (WIN32)
-  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS pi_win_proxy_loader)
+  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_win_proxy_loader)
 endif()
 
 if (TARGET sycl-prof)
@@ -477,8 +489,8 @@ if("cuda" IN_LIST SYCL_ENABLE_PLUGINS)
       "CUDA support requires adding \"libclc\" to the CMake argument \"LLVM_ENABLE_PROJECTS\"")
   endif()
 
-  add_dependencies(sycl-toolchain pi_cuda)
-  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS pi_cuda)
+  add_dependencies(sycl-toolchain ur_adapter_cuda)
+  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_adapter_cuda)
 endif()
 
 if("hip" IN_LIST SYCL_ENABLE_PLUGINS)
@@ -494,8 +506,8 @@ if("hip" IN_LIST SYCL_ENABLE_PLUGINS)
       "HIP support requires adding \"lld\" to the CMake argument \"LLVM_ENABLE_PROJECTS\"")
   endif()
 
-  add_dependencies(sycl-toolchain pi_hip)
-  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS pi_hip)
+  add_dependencies(sycl-toolchain ur_adapter_hip)
+  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_adapter_hip)
 endif()
 
 # Use it as fake dependency in order to force another command(s) to execute.

sycl/cmake/modules/AddSYCL.cmake

@@ -34,6 +34,8 @@ function(add_sycl_library LIB_NAME TYPE)
   add_common_options(${LIB_NAME})
 endfunction()
 
+# current ur adapter dependency managing is a bit hacky, we should try to copy
+# this as closely as possible instead
 function(add_sycl_plugin PLUGIN_NAME)
   cmake_parse_arguments("ARG"
     ""

sycl/cmake/modules/AddSYCLUnitTest.cmake

@@ -72,9 +72,12 @@ macro(add_sycl_unittest test_dirname link_variant)
     PRIVATE
       LLVMTestingSupport
       OpenCL-Headers
+      unified-runtime::mock
       ${SYCL_LINK_LIBS}
     )
 
+  add_dependencies(${test_dirname} ur_adapter_mock)
+
   if(SYCL_ENABLE_KERNEL_FUSION)
     target_link_libraries(${test_dirname} PRIVATE sycl-fusion)
   endif(SYCL_ENABLE_KERNEL_FUSION)

sycl/plugins/unified_runtime/CMakeLists.txt renamed to sycl/cmake/modules/FetchUnifiedRuntime.cmake

@@ -1,5 +1,13 @@
-# PI Unified Runtime plugin library.
-#
+# Either fetches UR from the appropriate repo or sets up variables based on user
+# preference.
+
+# TODO: taken from sycl/plugins/CMakeLists.txt - maybe we should handle this
+# within UR (although it is an obscure warning that the build system here
+# seems to specifically enable)
+if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|IntelLLVM" )
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-covered-switch-default")
+endif()
+
 
 # Options to override the default behaviour of the FetchContent to include UR
 # source code.
@@ -15,8 +23,11 @@ option(SYCL_PI_UR_USE_FETCH_CONTENT
 set(SYCL_PI_UR_SOURCE_DIR
   "" CACHE PATH "Path to root of Unified Runtime repository")
 
-# Override default to enable building tests from unified-runtime
-set(UR_BUILD_TESTS OFF CACHE BOOL "Build unit tests.")
+# Here we override the defaults to disable building tests from unified-runtime
+set(UR_BUILD_EXAMPLES OFF CACHE BOOL "Build example applications." FORCE)
+set(UR_BUILD_TESTS OFF CACHE BOOL "Build unit tests." FORCE)
+set(UR_BUILD_XPTI_LIBS OFF)
+set(UR_ENABLE_TRACING ON)
 
 if("level_zero" IN_LIST SYCL_ENABLE_PLUGINS)
   set(UR_BUILD_ADAPTER_L0 ON)
@@ -26,6 +37,9 @@ if("cuda" IN_LIST SYCL_ENABLE_PLUGINS)
 endif()
 if("hip" IN_LIST SYCL_ENABLE_PLUGINS)
   set(UR_BUILD_ADAPTER_HIP ON)
+  if (SYCL_ENABLE_KERNEL_FUSION)
+    set(UR_ENABLE_COMGR ON)
+  endif()
 endif()
 if("opencl" IN_LIST SYCL_ENABLE_PLUGINS)
   set(UR_BUILD_ADAPTER_OPENCL ON)
@@ -99,13 +113,13 @@ if(SYCL_PI_UR_USE_FETCH_CONTENT)
   endfunction()
 
   set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git")
-  # commit 2baf095188b235bb2b0a0140f0187d2041aef4b0
-  # Merge: 3d8fe8d2 58f85278
+  # commit b7b0c8b3d17aa7d511c67ec219d58091d07cfa60
+  # Merge: 2baf0951 5b8936da
   # Author: Piotr Balcer <[email protected]>
-  # Date:   Fri Jul 26 12:06:22 2024 +0200
-  #     Merge pull request #1900 from kswiecicki/umf-version-bump
-  #     Bump UMF version
-  set(UNIFIED_RUNTIME_TAG 58f85278a4ebf37742dd10afb3350580b0b1d9d7)
+  # Date:   Fri Jul 26 15:48:04 2024 +0200
+  #     Merge pull request #1903 from kswiecicki/umf-version-bump
+  #     Bump UMF version again
+  set(UNIFIED_RUNTIME_TAG b7b0c8b3d17aa7d511c67ec219d58091d07cfa60)
 
   set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "EXAMPLES")
   # Due to the use of dependentloadflag and no installer for UMF and hwloc we need
@@ -210,86 +224,69 @@ target_include_directories(UnifiedRuntime-Headers
 
 find_package(Threads REQUIRED)
 
-set(UNIFIED_RUNTIME_PLUGIN_ARGS
-  SOURCES
-    # These are short-term shared with Unified Runtime
-    # The two plugins define a few things differently so must
-    # be built separately. This difference is spelled in
-    # their "ur_bindings.hpp" files.
-    "ur_bindings.hpp"
-    "pi2ur.hpp"
-    # These below belong to Unified Runtime PI Plugin only
-    "pi_unified_runtime.hpp"
-    "pi_unified_runtime.cpp"
-  LIBRARIES
-    Threads::Threads
-    UnifiedRuntimeLoader
-    UnifiedRuntime-Headers
-    UnifiedRuntimeCommon
-  INCLUDE_DIRS
-    "${UNIFIED_RUNTIME_SRC_INCLUDE_DIR}"
-    "${UNIFIED_RUNTIME_COMMON_INCLUDE_DIR}"
-    "${SYCL_SOURCE_DIR}/source" # for compiler.hpp
-)
-
-# We need for #include <ze_api.h> in common.h
-if("level_zero" IN_LIST SYCL_ENABLE_PLUGINS)
-  list(APPEND UNIFIED_RUNTIME_PLUGIN_ARGS LevelZeroLoader-Headers)
-endif()
-
-if("opencl" IN_LIST SYCL_ENABLE_PLUGINS)
-  list(APPEND UNIFIED_RUNTIME_PLUGIN_ARGS OpenCL-ICD)
-endif()
-
-add_sycl_plugin(unified_runtime ${UNIFIED_RUNTIME_PLUGIN_ARGS})
-
 if(TARGET UnifiedRuntimeLoader)
-  set_target_properties(hello_world PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1)
   # Install the UR loader.
   # TODO: this is piggy-backing on the existing target component level-zero-sycl-dev
   # When UR is moved to its separate repo perhaps we should introduce new component,
   # e.g. unified-runtime-sycl-dev.
+  # See github issue #14598
   install(TARGETS ur_loader
     LIBRARY DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev
     ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev
     RUNTIME DESTINATION "bin" COMPONENT level-zero-sycl-dev
   )
 endif()
 
-# Install the UR adapters too
-if("level_zero" IN_LIST SYCL_ENABLE_PLUGINS)
-  add_dependencies(sycl-runtime-libraries ur_adapter_level_zero)
+add_custom_target(UnifiedRuntimeAdapters)
 
-  # Install the UR adapters too
-  install(TARGETS ur_adapter_level_zero
-    LIBRARY DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev
-    ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev
-    RUNTIME DESTINATION "bin" COMPONENT level-zero-sycl-dev
+function(add_sycl_ur_adapter NAME)
+  add_dependencies(UnifiedRuntimeAdapters ur_adapter_${NAME})
+
+  install(TARGETS ur_adapter_${NAME}
+    LIBRARY DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT ur_adapter_${NAME}
+    RUNTIME DESTINATION "bin" COMPONENT ur_adapter_${NAME})
+
+  set(manifest_file
+    ${CMAKE_CURRENT_BINARY_DIR}/install_manifest_ur_adapter_${NAME}.txt)
+  add_custom_command(OUTPUT ${manifest_file}
+    COMMAND "${CMAKE_COMMAND}"
+    "-DCMAKE_INSTALL_COMPONENT=ur_adapter_${NAME}"
+    -P "${CMAKE_BINARY_DIR}/cmake_install.cmake"
+    COMMENT "Deploying component ur_adapter_${NAME}"
+    USES_TERMINAL
   )
+  add_custom_target(install-sycl-ur-adapter-${NAME}
+    DEPENDS ${manifest_file} ur_adapter_${NAME}
+  )
+
+  set_property(GLOBAL APPEND PROPERTY
+    SYCL_TOOLCHAIN_INSTALL_COMPONENTS ur_adapter_${NAME})
+endfunction()
+
+if("level_zero" IN_LIST SYCL_ENABLE_PLUGINS)
+  add_sycl_ur_adapter(level_zero)
+
+  # TODO: L0 adapter does other... things in its cmake - make sure they get
+  # added to the new build system
 endif()
 
 if("cuda" IN_LIST SYCL_ENABLE_PLUGINS)
-  add_dependencies(sycl-runtime-libraries ur_adapter_cuda)
+  add_sycl_ur_adapter(cuda)
 endif()
 
 if("hip" IN_LIST SYCL_ENABLE_PLUGINS)
-  add_dependencies(sycl-runtime-libraries ur_adapter_hip)
+  add_sycl_ur_adapter(hip)
 endif()
 
-if ("opencl" IN_LIST SYCL_ENABLE_PLUGINS)
-    add_dependencies(sycl-runtime-libraries ur_adapter_opencl)
-
-    # Install the UR adapters too
-    install(TARGETS ur_adapter_opencl
-      LIBRARY DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev
-      ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev
-      RUNTIME DESTINATION "bin" COMPONENT level-zero-sycl-dev
-    )
+if("opencl" IN_LIST SYCL_ENABLE_PLUGINS)
+  add_sycl_ur_adapter(opencl)
 endif()
 
-if ("native_cpu" IN_LIST SYCL_ENABLE_PLUGINS)
-    add_dependencies(sycl-runtime-libraries ur_adapter_native_cpu)
+if("native_cpu" IN_LIST SYCL_ENABLE_PLUGINS)
+  add_sycl_ur_adapter(native_cpu)
 
+  # Deal with OCK option
+  option(NATIVECPU_USE_OCK "Use the oneAPI Construction Kit for Native CPU" ON)
 
   if(NATIVECPU_USE_OCK)
     message(STATUS "Compiling Native CPU adapter with OCK support.")
@@ -300,8 +297,6 @@ if ("native_cpu" IN_LIST SYCL_ENABLE_PLUGINS)
   endif()
 endif()
 
-add_dependencies(sycl-runtime-libraries ur_umf)
-
 # TODO: this is piggy-backing on the existing target component level-zero-sycl-dev
 install(TARGETS umf
   LIBRARY DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT level-zero-sycl-dev

sycl/doc/EnvironmentVariables.md

@@ -196,7 +196,7 @@ variables in production code.</span>
 | Environment variable | Values | Description |
 | -------------------- | ------ | ----------- |
 | `SYCL_PREFER_UR` | Integer | If non-0 then run through Unified Runtime if desired backend is supported there. Default is 0.  |
-| `SYCL_PI_TRACE` | Described [below](#sycl_pi_trace-options)  | Enable specified level of tracing for PI. |
+| `SYCL_UR_TRACE` | Integer | If non-0 then enable Unified Runtime tracing. Default is 0. |
 | `SYCL_QUEUE_THREAD_POOL_SIZE` | Positive integer | Number of threads in thread pool of queue. |
 | `SYCL_DEVICELIB_NO_FALLBACK` | Any(\*) | Disable loading and linking of device library images |
 | `SYCL_PRINT_EXECUTION_GRAPH` | Described [below](#sycl_print_execution_graph-options) | Print execution graph to DOT text file. |
@@ -231,16 +231,6 @@ variables in production code.</span>
 | after_addHostAcc | print graph after addHostAccessor method |
 | always | print graph before and after each of the above methods |
 
-### `SYCL_PI_TRACE` Options
-
-`SYCL_PI_TRACE` accepts a bit-mask. Supported tracing levels are in the table below
-
-| Option | Description |
-| ------ | ----------- |
-| 1 | Enable basic tracing, which is tracing of PI plugins/devices discovery |
-| 2 | Enable tracing of the PI calls |
-| -1 | Enable all levels of tracing |
-
 ## Debugging variables for Level Zero Plugin
 
 :warning: **Warning:** <span style="color:red">the environment variables

sycl/doc/design/PluginInterface.md

@@ -71,14 +71,12 @@ with arguments and returned values.
 SYCL_PI_TRACE=-1 lists all PI Traces above and more debug messages.
 
 #### Plugin binary interface
-Plugins should implement all the Interface APIs required for the PI Version
-it supports. There is [pi.def](../../include/sycl/detail/pi.def)/
-[pi.h](../../include/sycl/detail/pi.h) file listing all PI API names that
-can be called by the specific version of Plugin Interface.
-It exports a function - "piPluginInit" that returns the plugin details and
-function pointer table containing the list of pointers to implemented Interface
-Functions defined in pi.h.
-In the future, this document will list the minimum set of Interface APIs
+Plugins should implement all the Interface APIs required for the PI Version it
+supports. There is pi.def/ pi.h file listing all PI API names that can be
+called by the specific version of Plugin Interface. It exports a function -
+"piPluginInit" that returns the plugin details and function pointer table
+containing the list of pointers to implemented Interface Functions defined in
+pi.h. In the future, this document will list the minimum set of Interface APIs
 to be supported by Plugins. This will also require adding functionality to SYCL
 Runtime to work with such limited functionality plugins.
 
@@ -149,8 +147,7 @@ into
 - **Interoperability API** which allows interoperability with underlying
 runtimes such as OpenCL.
 
-See [pi.h](../../include/sycl/detail/pi.h) header for the full list and
-descriptions of PI APIs.
+See pi.h header for the full list and descriptions of PI APIs.
 
 ### The Core OpenCL-based PI APIs