[AUTO] Check if actual device cached blob exists before model compilation (#29636)

### Details:
- Enable AUTO to check if cache blob existed at the beginning of
compilation.
- Disable startup fallback and runtime fallback if any cache blob
existed for actual device
- ~~Add an new property for AUTO to indicate whether the core is
querying the cache ability from virtual device.~~

### Tickets:
 - CVS-164790

---------

Co-authored-by: Wanglei Shen <[email protected]>

Author: yangwang201911

src/inference/src/dev/core_impl.cpp

@@ -1456,11 +1456,9 @@ bool ov::CoreImpl::device_supports_internal_property(const ov::Plugin& plugin, c
 }
 
 bool ov::CoreImpl::device_supports_model_caching(const ov::Plugin& plugin, const ov::AnyMap& arguments) const {
-    ov::AnyMap properties;
-    if (arguments.count(ov::device::priorities.name())) {
-        properties[ov::device::priorities.name()] = arguments.at(ov::device::priorities.name()).as<std::string>();
-    }
-    return plugin.supports_model_caching(properties);
+    ov::AnyMap properties_to_virtual_dev = arguments.empty() ? ov::AnyMap{ov::device::priorities("")} : arguments;
+    return ov::is_virtual_device(plugin.get_name()) ? plugin.supports_model_caching(properties_to_virtual_dev)
+                                                    : plugin.supports_model_caching();
 }
 
 bool ov::CoreImpl::device_supports_cache_dir(const ov::Plugin& plugin) const {

src/plugins/auto/src/auto_schedule.cpp

@@ -135,49 +135,25 @@ void AutoSchedule::init() {
     auto customize_helper_context_from_cache_setting = [this](bool is_actual_cpu,
                                                               AutoCompileContext m_compile_context[],
                                                               ScheduleContext::Ptr& m_context) {
+        m_compile_context[CPU].m_is_enabled = true;
         const auto cpu_iter = deviceChecker().check_and_return_if_device_in_list("CPU", m_context->m_device_priorities);
-        if (cpu_iter == m_context->m_device_priorities.end()) {
+        if (cpu_iter == m_context->m_device_priorities.end() || is_actual_cpu) {
             m_compile_context[CPU].m_is_enabled = false;
             return;
         }
-        m_compile_context[CPU].m_is_enabled = true;
-        if (!is_actual_cpu) {
-            const auto& device = m_compile_context[ACTUALDEVICE].m_device_info.device_name;
-            auto& device_config = m_compile_context[ACTUALDEVICE].m_device_info.config;
-            std::string cache_dir = device_config.count(ov::cache_dir.name())
-                                        ? device_config[ov::cache_dir.name()].as<std::string>()
-                                        : m_context->m_ov_core->get_property("", ov::cache_dir);
-
-            if (m_context->m_startup_fallback && !cache_dir.empty()) {
-                const auto properties =
-                    m_context->m_ov_core->create_compile_config(ov::DeviceIDParser(device).get_device_name(),
-                                                                device_config);
-                std::string blobId;
-                if (m_context->m_model)
-                    blobId = ov::ModelCache::compute_hash(std::const_pointer_cast<const ov::Model>(m_context->m_model),
-                                                          properties);
-                else
-                    blobId = ov::ModelCache::compute_hash(m_context->m_model_path, properties);
-                std::string cached_model_path = ov::util::make_path(cache_dir, blobId + ".blob");
-                m_compile_context[CPU].m_is_enabled = !ov::util::file_exists(cached_model_path);
-                LOG_DEBUG_TAG("device: %s %s cached blob: %s ",
-                              device.c_str(),
-                              m_compile_context[CPU].m_is_enabled ? "not found" : "found",
-                              cached_model_path.c_str());
-            }
-        }
-        if (m_compile_context[CPU].m_is_enabled) {
-            m_compile_context[CPU].m_device_info = *cpu_iter;
-            m_compile_context[CPU].m_device_info.config[ov::hint::performance_mode.name()] =
-                ov::hint::PerformanceMode::LATENCY;
-            if (m_compile_context[ACTUALDEVICE].m_device_info.config.count(ov::cache_dir.name()) &&
-                (m_context->m_startup_fallback || m_context->m_runtime_fallback)) {
-                m_compile_context[CPU].m_device_info.config[ov::cache_dir.name()] = "";
-                LOG_INFO_TAG("Clear cache dir setting for CPU accelerator");
-            }
-            m_compile_context[CPU].m_worker_name = "CPU_HELP";
-            LOG_INFO_TAG("will load CPU for accelerator");
+        m_compile_context[CPU].m_device_info = *cpu_iter;
+        m_compile_context[CPU].m_device_info.config[ov::hint::performance_mode.name()] =
+            ov::hint::PerformanceMode::LATENCY;
+        std::string cache_dir =
+            m_compile_context[ACTUALDEVICE].m_device_info.config.count(ov::cache_dir.name())
+                ? m_compile_context[ACTUALDEVICE].m_device_info.config[ov::cache_dir.name()].as<std::string>()
+                : m_context->m_ov_core->get_property("", ov::cache_dir);
+        if (!cache_dir.empty() && (m_context->m_startup_fallback || m_context->m_runtime_fallback)) {
+            m_compile_context[CPU].m_device_info.config[ov::cache_dir.name()] = "";
+            LOG_INFO_TAG("Clear cache dir setting for CPU accelerator");
         }
+        m_compile_context[CPU].m_worker_name = "CPU_HELP";
+        LOG_INFO_TAG("will load CPU for accelerator");
     };
     if (m_compile_context[ACTUALDEVICE].m_is_enabled) {
         LOG_INFO_TAG("select device:%s", m_compile_context[ACTUALDEVICE].m_device_info.device_name.c_str());

src/plugins/auto/src/plugin.cpp

@@ -19,6 +19,8 @@
 #include "openvino/runtime/device_id_parser.hpp"
 #include "openvino/runtime/internal_properties.hpp"
 #include "openvino/runtime/iremote_context.hpp"
+#include "openvino/runtime/compilation_context.hpp"
+#include "openvino/util/file_util.hpp"
 #include "plugin.hpp"
 #include "auto_schedule.hpp"
 #include "auto_compiled_model.hpp"
@@ -78,7 +80,6 @@ namespace auto_plugin {
 std::shared_ptr<std::mutex> Plugin::m_mtx = std::make_shared<std::mutex>();
 std::shared_ptr<std::map<unsigned int, std::list<std::string>>> Plugin::m_priority_map =
     std::make_shared<std::map<unsigned int, std::list<std::string>>>();
-
 ov::SoPtr<ov::IRemoteContext> Plugin::create_context(const ov::AnyMap& remote_properties) const {
     OPENVINO_NOT_IMPLEMENTED;
 }
@@ -304,14 +305,17 @@ ov::Any Plugin::get_property(const std::string& name, const ov::AnyMap& argument
     } else if (name == ov::device::full_name) {
         return decltype(ov::device::full_name)::value_type {get_device_name()};
     } else if (name == ov::device::capabilities.name()) {
-        std::vector<std::string> device_list = arguments.count(ov::device::priorities.name())
-                                                   ? m_plugin_config.parse_priorities_devices(
-                                                         arguments.at(ov::device::priorities.name()).as<std::string>())
+        if (arguments.count(ov::device::priorities.name())) {
+            // By default, all devices are assumed to support caching ability when Core checks caching ability for AUTO.
+            return std::vector<std::string>{ov::device::capability::EXPORT_IMPORT};
+        }
+        std::string priorities = m_plugin_config.get_property(ov::device::priorities.name()).as<std::string>();
+        std::vector<std::string> device_list = !priorities.empty()
+                                                   ? m_plugin_config.parse_priorities_devices(priorities)
                                                    : get_core()->get_available_devices();
+
         std::vector<std::string> capabilities;
         for (auto const& device : device_list) {
-            if (device[0] == '-')
-                continue;
             try {
                 auto dev_capabilities = get_core()->get_property(device, ov::device::capabilities);
                 capabilities.insert(capabilities.end(), dev_capabilities.begin(), dev_capabilities.end());
@@ -409,7 +413,16 @@ std::shared_ptr<ov::ICompiledModel> Plugin::compile_model_impl(const std::string
     // and set filter configure
     auto auto_s_context = std::make_shared<ScheduleContext>();
     ov::AnyMap filter_property;
-    auto str_devices = get_device_list(full_property);
+    auto str_devices = get_device_list(full_property, model, model_path);
+    // in case startup or runtime fallback is set caused by cache blob checking, we need to set the property
+    if (full_property.count(ov::intel_auto::enable_startup_fallback.name())) {
+        load_config.set_property(ov::intel_auto::enable_startup_fallback(
+            full_property.at(ov::intel_auto::enable_startup_fallback.name()).as<bool>()));
+    }
+    if (full_property.count(ov::intel_auto::enable_runtime_fallback.name())) {
+        load_config.set_property(ov::intel_auto::enable_runtime_fallback(
+            full_property.at(ov::intel_auto::enable_runtime_fallback.name()).as<bool>()));
+    }
     // fill in the context for auto
     if (load_config.get_property(ov::enable_profiling)) {
         filter_property.insert({ov::enable_profiling(true)});
@@ -694,7 +707,9 @@ void Plugin::register_priority(const unsigned int& priority, const std::string&
     }
 }
 
-std::string Plugin::get_device_list(const ov::AnyMap& properties) const {
+std::string Plugin::get_device_list(ov::AnyMap& properties,
+                                    const std::shared_ptr<const ov::Model>& model,
+                                    const std::string& model_path) const {
     std::string all_devices;
     std::string device_architecture;
     auto device_list_config = properties.find(ov::device::priorities.name());
@@ -708,10 +723,56 @@ std::string Plugin::get_device_list(const ov::AnyMap& properties) const {
         return "";
     };
     std::vector<std::string> devices_merged;
+    bool enable_startup_cpu = properties.count(ov::intel_auto::enable_startup_fallback.name())
+                                  ? properties.at(ov::intel_auto::enable_startup_fallback.name()).as<bool>()
+                                  : true;
+    bool enable_runtime_cpu = properties.count(ov::intel_auto::enable_runtime_fallback.name())
+                                  ? properties.at(ov::intel_auto::enable_runtime_fallback.name()).as<bool>()
+                                  : true;
+    bool is_cumulative_tput =
+        get_device_name() != "AUTO" ||
+        (properties.count(ov::hint::performance_mode.name()) &&
+         properties.at(ov::hint::performance_mode.name()).as<std::string>() == "CUMULATIVE_THROUGHPUT");
     if (device_list_config != properties.end() && !(device_list_config->second.as<std::string>().empty())) {
         auto priorities = device_list_config->second;
         // parsing the string and splitting the comma-separated tokens
-        std::vector<std::string> devices_to_be_merged = m_plugin_config.parse_priorities_devices(priorities.as<std::string>());
+        std::vector<std::string> devices_to_be_merged =
+            m_plugin_config.parse_priorities_devices(priorities.as<std::string>());
+        std::size_t num_blob_files = 0;
+        std::string cache_dir = properties.count(ov::cache_dir.name())
+                                    ? properties.at(ov::cache_dir.name()).as<std::string>()
+                                    : get_core()->get_property("", ov::cache_dir);
+        bool if_need_cache_check =
+            !is_cumulative_tput && enable_startup_cpu && (model || !model_path.empty()) && !cache_dir.empty();
+        if (if_need_cache_check) {
+            for (auto&& device : devices_to_be_merged) {
+                ov::DeviceIDParser parsed{device};
+                if (parsed.get_device_name().find("CPU") != std::string::npos)
+                    continue;
+                // check if cached model exists for other devices
+                auto dev_properties = get_core()->get_supported_property(parsed.get_device_name(), properties);
+                dev_properties = get_core()->create_compile_config(parsed.get_device_name(), dev_properties);
+                std::string blobId;
+
+                if (model)
+                    blobId = ov::ModelCache::compute_hash(std::const_pointer_cast<const ov::Model>(model),
+                                                          dev_properties);
+                else
+                    blobId = ov::ModelCache::compute_hash(model_path, dev_properties);
+                std::string cached_model_path = ov::util::make_path(cache_dir, blobId + ".blob");
+                bool is_blob_file_exist = ov::util::file_exists(cached_model_path);
+                num_blob_files += is_blob_file_exist;
+                LOG_DEBUG_TAG("device: %s %s cached blob: %s ",
+                              device.c_str(),
+                              is_blob_file_exist ? "found" : "not found",
+                              cached_model_path.c_str());
+            }
+
+            if (enable_startup_cpu && num_blob_files) {
+                LOG_DEBUG_TAG("Disabling CPU fallback as a cached blob file was found for a device in the candidate "
+                              "list. AUTO will work as pass-through mode.");
+            }
+        }
         std::vector<std::string> devices_to_be_deleted(devices_to_be_merged.size());
         const auto& iterDel = std::copy_if(devices_to_be_merged.begin(),
                                            devices_to_be_merged.end(),
@@ -765,17 +826,16 @@ std::string Plugin::get_device_list(const ov::AnyMap& properties) const {
                 std::vector<std::string> device_list = {};
                 try {
                     if (parsed.get_device_name().find("CPU") != std::string::npos) {
-                        bool enable_startup_cpu =
-                            properties.count(ov::intel_auto::enable_startup_fallback.name())
-                                ? properties.at(ov::intel_auto::enable_startup_fallback.name()).as<bool>()
-                                : true;
-                        bool enable_runtime_cpu =
-                            properties.count(ov::intel_auto::enable_runtime_fallback.name())
-                                ? properties.at(ov::intel_auto::enable_runtime_fallback.name()).as<bool>()
-                                : true;
-                        // Skip to load CPU device if both startup and runtime fallback are disabled
-                        if (!enable_startup_cpu && !enable_runtime_cpu)
+                        // Disable CPU if any blob files found
+                        if (num_blob_files) {
+                            properties[ov::intel_auto::enable_startup_fallback.name()] = false;
+                            properties[ov::intel_auto::enable_runtime_fallback.name()] = false;
+                            continue;
+                        }
+                        // Disable CPU if enable_startup_cpu and enable_runtime_cpu are both disabled
+                        if (!enable_startup_cpu && !enable_runtime_cpu) {
                             continue;
+                        }
                     }
                     auto device_id_list = get_core()
                                               ->get_property(parsed.get_device_name(), ov::available_devices.name(), {})

src/plugins/auto/src/plugin.hpp

@@ -45,10 +45,12 @@ class Plugin : public ov::IPlugin {
     MOCKTESTMACRO std::vector<auto_plugin::DeviceInformation> parse_meta_devices(const std::string & devices_requests_cfg,
                                                                                  const ov::AnyMap& properties) const;
 
-    MOCKTESTMACRO std::string get_device_list(const ov::AnyMap& properties) const;
+    MOCKTESTMACRO std::string get_device_list(ov::AnyMap& properties,
+                                              const std::shared_ptr<const ov::Model>& model = nullptr,
+                                              const std::string& model_path = {}) const;
 
     MOCKTESTMACRO std::list<DeviceInformation> get_valid_device(const std::vector<DeviceInformation>& meta_devices,
-                                                   const std::string& model_precision = "FP32") const;
+                                                                const std::string& model_precision = "FP32") const;
 
     MOCKTESTMACRO DeviceInformation select_device(const std::vector<DeviceInformation>& meta_devices,
                                                  const std::string& model_precision = "FP32",

src/plugins/auto/tests/functional/behavior/caching_test.cpp

@@ -43,8 +43,8 @@ TEST_F(AutoFuncTests, load_cached_model_to_actual_device_and_disable_CPU_acceler
                                                   ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)});
     }
     // No cached model for actual device
-    // will cache model for both actual device and CPU plugin
-    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 2);
+    // will only cache model only for actual device. CVS-141026
+    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 1);
     ov::test::utils::removeFilesWithExt(cache_path, "blob");
     {
         auto compiled_model = core.compile_model(
@@ -70,8 +70,8 @@ TEST_F(AutoFuncTests, load_cached_model_to_actual_device_and_disable_CPU_acceler
                                                   ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)});
     }
     // model hash id changed for actual device
-    // will cache model for both actual device and CPU as accelerator
-    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 3);
+    // will cache model for actual device with new device id
+    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 2);
     core.set_property(ov::cache_dir(""));
 }
 
@@ -89,8 +89,8 @@ TEST_F(AutoFuncTests, load_model_path_to_actual_device_and_disable_CPU_accelerat
                                                   ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)});
     }
     // No cached model for actual device
-    // will cache model for both actual device and CPU plugin
-    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 2);
+    // will only cache model only for actual device. CVS-141026
+    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 1);
     ov::test::utils::removeFilesWithExt(cache_path, "blob");
     {
         auto compiled_model = core.compile_model(
@@ -116,8 +116,8 @@ TEST_F(AutoFuncTests, load_model_path_to_actual_device_and_disable_CPU_accelerat
                                                   ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)});
     }
     // model hash id changed for actual device
-    // will cache model for both actual device and CPU as accelerator
-    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 3);
+    // will cache model for actual device with new device id
+    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 2);
     core.set_property(ov::cache_dir(""));
     ov::test::utils::removeIRFiles(m_xml_path, m_bin_path);
 }
@@ -132,8 +132,8 @@ TEST_F(AutoFuncTests, load_cached_model_to_actual_device_and_disable_CPU_acceler
                                                   ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)});
     }
     // No cached model for actual device
-    // will cache model for both actual device and CPU plugin
-    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 2);
+    // will only cache model only for actual device. CVS-141026
+    ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 1);
     ov::test::utils::removeFilesWithExt(cache_path, "blob");
     {
         auto compiled_model = core.compile_model(
@@ -160,7 +160,7 @@ TEST_F(AutoFuncTests, load_cached_model_to_actual_device_and_disable_CPU_acceler
                                                   ov::intel_auto::enable_startup_fallback(false)});
     }
     // model hash id changed for actual device
-    // will cache 2 models for actual device and no cached model for CPU
+    // will cache model for actual device with new device id
     ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 2);
     core.set_property(ov::cache_dir(""));
 }