diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 48d1d0ca4a082..8ead9bda2f653 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5791,7 +5791,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   // Enable local accessor to shared memory pass for SYCL.
-  if (isa<BackendJobAction>(JA) && IsSYCL) {
+  if (isa<BackendJobAction>(JA) && IsSYCLOffloadDevice &&
+      (Triple.isNVPTX() || Triple.isAMDGCN())) {
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-sycl-enable-local-accessor");
   }
diff --git a/clang/test/Driver/sycl-local-accessor-opt.cpp b/clang/test/Driver/sycl-local-accessor-opt.cpp
index 855ef09e801e9..154eee5178d8c 100644
--- a/clang/test/Driver/sycl-local-accessor-opt.cpp
+++ b/clang/test/Driver/sycl-local-accessor-opt.cpp
@@ -3,8 +3,11 @@
 // REQUIRES: clang-driver
 
 // RUN:   %clang -fsycl -### %s 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHECK-NO-OPT %s
-// CHECK-NO-OPT-NOT: "-sycl-enable-local-accessor"
+// RUN:   | FileCheck -check-prefix=OPT-CHECK %s
+
+// RUN:   %clang -fsycl -S -### %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=OPT-CHECK %s
+// OPT-CHECK-NOT: "-sycl-enable-local-accessor"
 
 // RUN:   %clang -fsycl -fsycl-targets=nvptx64-nvidia-cuda -### %s 2>&1 \
 // RUN:   | FileCheck %s