Skip to content

Copy weights file to epctx output directory #648

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: ovep-develop
Choose a base branch
from
+45 −17
Draft

Copy weights file to epctx output directory #648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions onnxruntime/core/providers/openvino/backend_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,18 +84,21 @@ BackendManager::BackendManager(SessionContext& session_context,
std::string device_type = session_context_.device_type;

auto& sw = shared_context_.shared_weights;
if (sw.external_weight_filename.empty() && !sw.metadata.empty()) {
// Reasonable assumption that all metadata entries have the same external file location
sw.external_weight_filename = sw.metadata.begin()->second.location;
}

if (session_context_.so_share_ep_contexts) {
std::filesystem::path weight_filename = session_context_.onnx_model_path_name.parent_path();
if (sw.external_weight_filename.empty() && !sw.metadata.empty()) {
// Reasonable assumption that all metadata entries have the same external file location
sw.external_weight_filename = sw.metadata.begin()->second.location;
auto weight_path = session_context_.GetNewWeightsFilePath(sw.external_weight_filename);
if (!std::filesystem::exists(weight_path)) {
weight_path = session_context_.GetModelDirectory() / sw.external_weight_filename;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment as below

}
weight_filename /= sw.external_weight_filename;
std::ifstream weight_file(weight_filename);

std::ifstream weight_file(weight_path);
if (weight_file) {
if (!sw.mapped_weights) {
sw.mapped_weights = std::make_unique<SharedContext::SharedWeights::WeightsFile>(weight_filename);
sw.mapped_weights = std::make_unique<SharedContext::SharedWeights::WeightsFile>(weight_path);
}
backend_utils::CreateOVTensors(session_context_.device_type, sw.metadata, *sw.mapped_weights);
}
Expand Down Expand Up @@ -241,7 +244,7 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
std::ofstream blob_file(blob_filename,
std::ios::out | std::ios::trunc | std::ios::binary);
if (!blob_file) {
ORT_THROW("Unable to open file for epctx model dump.");
ORT_THROW("Unable to open file for epctx model dump." + blob_filename.string());
}
compiled_model.export_model(blob_file);
model_blob_str = blob_filename.filename().string();
Expand Down Expand Up @@ -324,7 +327,7 @@ static bool IsQDQGraph(const onnxruntime::GraphViewer& graph_viewer) {
static void DumpOpenVINOEPModel([[maybe_unused]] const std::filesystem::path& onnx_model_path_name,
[[maybe_unused]] ONNX_NAMESPACE::ModelProto* model_proto,
[[maybe_unused]] const onnxruntime::Node& fused_node) {
#ifdef NOT_RELEASE
#ifdef NOT_RELEASE
if (openvino_ep::backend_utils::IsDebugEnabled()) {
auto model_name = onnx_model_path_name.empty() ? "unknown.onnx" : onnx_model_path_name.filename();

Expand Down
14 changes: 14 additions & 0 deletions onnxruntime/core/providers/openvino/contexts.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,20 @@ struct SessionContext : ProviderInfo {
mutable bool has_external_weights = false; // Value is set to mutable to modify from capability
const std::vector<uint32_t> OpenVINO_Version = {OPENVINO_VERSION_MAJOR, OPENVINO_VERSION_MINOR};
const std::string openvino_sdk_version = std::to_string(OPENVINO_VERSION_MAJOR) + "." + std::to_string(OPENVINO_VERSION_MINOR);

fs::path GetModelDirectory() const {
return onnx_model_path_name.parent_path();
}

fs::path GetEpContextOutputDirectory() const {
return so_context_file_path.empty() ? GetModelDirectory() : so_context_file_path.parent_path();
}

fs::path GetNewWeightsFilePath(fs::path external_weights_filename) const {
ORT_ENFORCE(!external_weights_filename.empty(), "External weights filename should not be empty.");
// Otherwise, use the provided external weights filename.
return GetEpContextOutputDirectory() / fs::path(external_weights_filename.filename().string() + "_weights.bin");
}
};

// Holds context specific to subgraph.
Expand Down
27 changes: 19 additions & 8 deletions onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,13 @@ common::Status OpenVINOExecutionProvider::Compile(
graph_body_viewer_0.DomainToVersionMap().at(kOnnxDomain);
}

const auto metadata_path = session_context_.GetEpContextOutputDirectory() / "metadata.bin";

// Temporary code to read metadata before it moves to the .bin
auto& metadata = shared_context_->shared_weights.metadata;
if (session_context_.so_share_ep_contexts && metadata.empty()) {
// Metadata is always read from model location, this could be a source or epctx model
fs::path metadata_filename = session_context_.onnx_model_path_name.parent_path() / "metadata.bin";
std::ifstream file(metadata_filename, std::ios::binary);
std::ifstream file(metadata_path, std::ios::binary);
if (file) {
file >> metadata;
}
Expand Down Expand Up @@ -174,20 +175,30 @@ common::Status OpenVINOExecutionProvider::Compile(
}

if (session_context_.so_share_ep_contexts) {
fs::path metadata_filename;
if (session_context_.so_context_file_path.empty()) {
metadata_filename = session_context_.onnx_model_path_name.parent_path() / "metadata.bin";
const auto& sw_path_filename = shared_context_->shared_weights.external_weight_filename;
fs::path new_weights_file_path = session_context_.GetNewWeightsFilePath(sw_path_filename);
fs::path original_weights_path = session_context_.GetModelDirectory() / sw_path_filename;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not quite right. There's an ORT option for setting the weights path when for whatever reason the relative path between model and weight was altered from when the model was generated. For example if an initializer in the model has location = "weights.bin" then the file weights.bin is expected to be in the same folder as the model. If the weights are instead in "path\weights.bin" then session option ep.context_model_external_initializers_file_name MUST be set to "path".

In summary this needs a

fs::path GetWeightsDirectory() const {
    return so_context_model_external_initializers_file_name.empty() ? GetModelDirectory() : so_so_context_model_external_initializers_file_name;
}

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on yesterday's discussion we're not going to deal with context_model_external_initializers_file_name in this PR


if (!std::filesystem::exists(new_weights_file_path)) {
try {
std::filesystem::create_hard_link(original_weights_path, new_weights_file_path);
} catch (const std::filesystem::filesystem_error& e) {
LOGS_DEFAULT(WARNING) << "Failed to create hard link: " << e.what() << " Falling back to copy.";
std::filesystem::copy_file(original_weights_path, new_weights_file_path);
}
} else {
metadata_filename = session_context_.so_context_file_path.parent_path() / "metadata.bin";
LOGS_DEFAULT(WARNING) << "Weights file already exists: " << new_weights_file_path.string() << " Link/Copy.";
}

// Metadata is generated only for shared contexts
// If saving metadata then save it to the provided path or ose the original model path
// If saving metadata then save it to the provided path or use the original model path
// Multiple calls to Compile() will update the metadata and for the last call
// the resulting file will contain the aggregated content
std::ofstream file(metadata_filename, std::ios::binary);
std::ofstream file(metadata_path, std::ios::binary);
if (file) {
file << metadata;
} else {
LOGS_DEFAULT(WARNING) << "Failed to write metadata to file: " << metadata_path.string();
}
}

Expand Down