Shader data can now be compiled into the binary.

chinmaygarde · dnfield · commit 0b63e2cc2c66 · 2022-04-27T15:57:33.000-07:00
There is no more need to figure out separate packaging of shader data.
diff --git a/impeller/playground/BUILD.gn b/impeller/playground/BUILD.gn
@@ -14,6 +14,8 @@ impeller_component("playground") {
   ]
 
   public_deps = [
+    "../entity:entity_shaders",
+    "../fixtures:shader_fixtures",
     "../renderer",
     "//flutter/testing",
     "//third_party/glfw",
diff --git a/impeller/playground/playground.mm b/impeller/playground/playground.mm
@@ -5,6 +5,8 @@
 #include <sstream>
 
 #include "flutter/fml/paths.h"
+#include "flutter/impeller/entity/entity_shaders.h"
+#include "flutter/impeller/fixtures/shader_fixtures.h"
 #include "flutter/testing/testing.h"
 #include "impeller/base/validation.h"
 #include "impeller/image/compressed_image.h"
@@ -28,27 +30,19 @@
 
 namespace impeller {
 
-static std::string ShaderLibraryDirectory() {
-  auto path_result = fml::paths::GetExecutableDirectoryPath();
-  if (!path_result.first) {
-    return {};
-  }
-  return fml::paths::JoinPaths({path_result.second, "shaders"});
-}
+static std::vector<std::shared_ptr<fml::Mapping>>
+ShaderLibraryMappingsForPlayground() {
+  return {
+      std::make_shared<fml::NonOwnedMapping>(impeller_entity_shaders_data,
+                                             impeller_entity_shaders_length),
+      std::make_shared<fml::NonOwnedMapping>(impeller_shader_fixtures_data,
+                                             impeller_shader_fixtures_length),
 
-static std::vector<std::string> ShaderLibraryPathsForPlayground() {
-  std::vector<std::string> paths;
-  paths.emplace_back(fml::paths::JoinPaths(
-      {ShaderLibraryDirectory(), "shader_fixtures.metallib"}));
-  paths.emplace_back(
-      fml::paths::JoinPaths({fml::paths::GetExecutableDirectoryPath().second,
-                             "shaders", "entity.metallib"}));
-  return paths;
+  };
 }
 
 Playground::Playground()
-    : renderer_(
-          std::make_shared<ContextMTL>(ShaderLibraryPathsForPlayground())) {}
+    : renderer_(ContextMTL::Create(ShaderLibraryMappingsForPlayground())) {}
 
 Playground::~Playground() = default;
 
@@ -170,10 +164,10 @@ static void PlaygroundKeyCallback(GLFWwindow* window,
   CompressedImage compressed_image(
       flutter::testing::OpenFixtureAsMapping(fixture_name));
   // The decoded image is immediately converted into RGBA as that format is
-  // known to be supported everywhere. For image sources that don't need 32 bit
-  // pixel strides, this is overkill. Since this is a test fixture we aren't
-  // necessarily trying to eke out memory savings here and instead favor
-  // simplicity.
+  // known to be supported everywhere. For image sources that don't need 32
+  // bit pixel strides, this is overkill. Since this is a test fixture we
+  // aren't necessarily trying to eke out memory savings here and instead
+  // favor simplicity.
   auto image = compressed_image.Decode().ConvertToRGBA();
   if (!image.IsValid()) {
     VALIDATION_LOG << "Could not find fixture named " << fixture_name;
diff --git a/impeller/renderer/backend/metal/context_mtl.h b/impeller/renderer/backend/metal/context_mtl.h
@@ -23,7 +23,11 @@ namespace impeller {
 class ContextMTL final : public Context,
                          public BackendCast<ContextMTL, Context> {
  public:
-  ContextMTL(const std::vector<std::string>& shader_libraries);
+  static std::shared_ptr<Context> Create(
+      const std::vector<std::string>& shader_library_paths);
+
+  static std::shared_ptr<Context> Create(
+      const std::vector<std::shared_ptr<fml::Mapping>>& shader_libraries_data);
 
   // |Context|
   ~ContextMTL() override;
@@ -41,6 +45,8 @@ class ContextMTL final : public Context,
   std::shared_ptr<AllocatorMTL> transients_allocator_;
   bool is_valid_ = false;
 
+  ContextMTL(id<MTLDevice> device, NSArray<id<MTLLibrary>>* shader_libraries);
+
   // |Context|
   bool IsValid() const override;
 
diff --git a/impeller/renderer/backend/metal/context_mtl.mm b/impeller/renderer/backend/metal/context_mtl.mm
@@ -14,36 +14,32 @@
 
 namespace impeller {
 
-static NSArray<id<MTLLibrary>>* ShaderLibrariesFromFiles(
-    id<MTLDevice> device,
-    const std::vector<std::string>& libraries_paths) {
-  NSMutableArray<id<MTLLibrary>>* found_libraries = [NSMutableArray array];
-  for (const auto& library_path : libraries_paths) {
-    if (!fml::IsFile(library_path)) {
-      VALIDATION_LOG << "Shader library does not exist at path '"
-                     << library_path << "'";
-      continue;
-    }
-    NSError* shader_library_error = nil;
-    auto library = [device newLibraryWithFile:@(library_path.c_str())
-                                        error:&shader_library_error];
-    if (!library) {
-      FML_LOG(ERROR) << "Could not create shader library: "
-                     << shader_library_error.localizedDescription.UTF8String;
-      continue;
-    }
-    [found_libraries addObject:library];
-  }
-  return found_libraries;
-}
-
-ContextMTL::ContextMTL(const std::vector<std::string>& libraries_paths)
-    : device_(::MTLCreateSystemDefaultDevice()) {
-  // Setup device.
+ContextMTL::ContextMTL(id<MTLDevice> device,
+                       NSArray<id<MTLLibrary>>* shader_libraries)
+    : device_(device) {
+  // Validate device.
   if (!device_) {
+    VALIDATION_LOG << "Could not setup valid Metal device.";
     return;
   }
 
+  // Setup the shader library.
+  {
+    if (shader_libraries == nil) {
+      VALIDATION_LOG << "Shader libraries were null.";
+      return;
+    }
+
+    // std::make_shared disallowed because of private friend ctor.
+    auto library = std::shared_ptr<ShaderLibraryMTL>(
+        new ShaderLibraryMTL(shader_libraries));
+    if (!library->IsValid()) {
+      VALIDATION_LOG << "Could not create valid Metal shader library.";
+      return;
+    }
+    shader_library_ = std::move(library);
+  }
+
   // Setup command queues.
   render_queue_ = device_.newCommandQueue;
   transfer_queue_ = device_.newCommandQueue;
@@ -55,18 +51,6 @@
   render_queue_.label = @"Impeller Render Queue";
   transfer_queue_.label = @"Impeller Transfer Queue";
 
-  // Setup the shader library.
-  {
-    // std::make_shared disallowed because of private friend ctor.
-    auto library = std::shared_ptr<ShaderLibraryMTL>(new ShaderLibraryMTL(
-        ShaderLibrariesFromFiles(device_, libraries_paths)));
-    if (!library->IsValid()) {
-      VALIDATION_LOG << "Could not create valid Metal shader library.";
-      return;
-    }
-    shader_library_ = std::move(library);
-  }
-
   // Setup the pipeline library.
   {  //
     pipeline_library_ =
@@ -96,6 +80,96 @@
   is_valid_ = true;
 }
 
+static NSArray<id<MTLLibrary>>* MTLShaderLibraryFromFilePaths(
+    id<MTLDevice> device,
+    const std::vector<std::string>& libraries_paths) {
+  NSMutableArray<id<MTLLibrary>>* found_libraries = [NSMutableArray array];
+  for (const auto& library_path : libraries_paths) {
+    if (!fml::IsFile(library_path)) {
+      VALIDATION_LOG << "Shader library does not exist at path '"
+                     << library_path << "'";
+      return nil;
+    }
+    NSError* shader_library_error = nil;
+    auto library = [device newLibraryWithFile:@(library_path.c_str())
+                                        error:&shader_library_error];
+    if (!library) {
+      FML_LOG(ERROR) << "Could not create shader library: "
+                     << shader_library_error.localizedDescription.UTF8String;
+      return nil;
+    }
+    [found_libraries addObject:library];
+  }
+  return found_libraries;
+}
+
+static NSArray<id<MTLLibrary>>* MTLShaderLibraryFromFileData(
+    id<MTLDevice> device,
+    const std::vector<std::shared_ptr<fml::Mapping>>& libraries_data) {
+  NSMutableArray<id<MTLLibrary>>* found_libraries = [NSMutableArray array];
+  for (const auto& library_data : libraries_data) {
+    if (library_data == nullptr) {
+      FML_LOG(ERROR) << "Shader library data was null.";
+      return nil;
+    }
+
+    __block auto data = library_data;
+
+    auto dispatch_data =
+        ::dispatch_data_create(library_data->GetMapping(),  // buffer
+                               library_data->GetSize(),     // size
+                               dispatch_get_main_queue(),   // queue
+                               ^() {
+                                 // We just need a reference.
+                                 data.reset();
+                               }  // destructor
+        );
+    if (!dispatch_data) {
+      FML_LOG(ERROR) << "Could not wrap shader data in dispatch data.";
+      return nil;
+    }
+
+    NSError* shader_library_error = nil;
+    auto library = [device newLibraryWithData:dispatch_data
+                                        error:&shader_library_error];
+    if (!library) {
+      FML_LOG(ERROR) << "Could not create shader library: "
+                     << shader_library_error.localizedDescription.UTF8String;
+      return nil;
+    }
+    [found_libraries addObject:library];
+  }
+  return found_libraries;
+}
+
+static id<MTLDevice> CreateMetalDevice() {
+  return ::MTLCreateSystemDefaultDevice();
+}
+
+std::shared_ptr<Context> ContextMTL::Create(
+    const std::vector<std::string>& shader_library_paths) {
+  auto device = CreateMetalDevice();
+  auto context = std::shared_ptr<ContextMTL>(new ContextMTL(
+      device, MTLShaderLibraryFromFilePaths(device, shader_library_paths)));
+  if (!context->IsValid()) {
+    FML_LOG(ERROR) << "Could not create Metal context.";
+    return nullptr;
+  }
+  return context;
+}
+
+std::shared_ptr<Context> ContextMTL::Create(
+    const std::vector<std::shared_ptr<fml::Mapping>>& shader_libraries_data) {
+  auto device = CreateMetalDevice();
+  auto context = std::shared_ptr<ContextMTL>(new ContextMTL(
+      device, MTLShaderLibraryFromFileData(device, shader_libraries_data)));
+  if (!context->IsValid()) {
+    FML_LOG(ERROR) << "Could not create Metal context.";
+    return nullptr;
+  }
+  return context;
+}
+
 ContextMTL::~ContextMTL() = default;
 
 bool ContextMTL::IsValid() const {
diff --git a/impeller/tools/impeller.gni b/impeller/tools/impeller.gni
@@ -100,6 +100,7 @@ template("impeller_shaders") {
   assert(defined(invoker.shaders), "Impeller shaders must be specified.")
   assert(defined(invoker.name), "Name of the shader library must be specified.")
 
+  base_target_name = target_name
   impellerc_target_name = "impellerc_$target_name"
   compiled_action_foreach(impellerc_target_name) {
     tool = "//flutter/impeller/compiler:impellerc"
@@ -192,9 +193,40 @@ template("impeller_shaders") {
     ]
   }
 
+  generate_embedder_data_sources = "embedded_data_gen_sources_$target_name"
+  action(generate_embedder_data_sources) {
+    metal_library_files = get_target_outputs(":$metal_library_target_name")
+    metal_library_file = metal_library_files[0]
+    inputs = [ metal_library_file ]
+    output_header = "$target_gen_dir/$base_target_name.h"
+    output_source = "$target_gen_dir/$base_target_name.c"
+    outputs = [
+      output_header,
+      output_source,
+    ]
+    args = [
+      "--symbol-name",
+      base_target_name,
+      "--output-header",
+      rebase_path(output_header),
+      "--output-source",
+      rebase_path(output_source),
+      "--source",
+      rebase_path(metal_library_file),
+    ]
+    script = "//flutter/impeller/tools/xxd.py"
+    deps = [ ":$metal_library_target_name" ]
+  }
+
+  shader_embedded_data_target_name = "embedded_data_$target_name"
+  source_set(shader_embedded_data_target_name) {
+    sources = get_target_outputs(":$generate_embedder_data_sources")
+    deps = [ ":$generate_embedder_data_sources" ]
+  }
+
   group(target_name) {
     public_deps = [
-      ":$metal_library_target_name",
+      ":$shader_embedded_data_target_name",
       ":$shader_glue_target_name",
     ]
   }
diff --git a/impeller/tools/xxd.py b/impeller/tools/xxd.py
@@ -0,0 +1,75 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+
+import argparse
+import errno
+import os
+import struct
+
+def MakeDirectories(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+# Dump the bytes of file into a C translation unit.
+# This can be used to embed the file contents into a binary.
+def Main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--symbol-name",
+                    type=str, required=True,
+                    help="The name of the symbol referencing the data.")
+  parser.add_argument("--output-header",
+                    type=str, required=True,
+                    help="The header file containing the symbol reference.")
+  parser.add_argument("--output-source",
+                    type=str, required=True,
+                    help="The source file containing the file bytes.")
+  parser.add_argument("--source",
+                    type=str, required=True,
+                    help="The source file whose contents to embed in the output source file.")
+
+  args = parser.parse_args()
+
+  assert(os.path.exists(args.source))
+
+  output_header = os.path.abspath(args.output_header)
+  output_source = os.path.abspath(args.output_source)
+
+  MakeDirectories(os.path.dirname(output_header))
+  MakeDirectories(os.path.dirname(output_source))
+
+  with open(args.source, "rb") as source, open(output_source, "w") as output:
+    data_len = 0
+    output.write(f"const unsigned char impeller_{args.symbol_name}_data[] =\n")
+    output.write("{\n")
+    while True:
+      byte = source.read(1)
+      if not byte:
+        break
+      data_len += 1
+      output.write(f"{ord(byte)},")
+    output.write("};\n")
+    output.write(f"const unsigned long impeller_{args.symbol_name}_length = {data_len};\n")
+
+  with open(output_header, "w") as output:
+    output.write("#pragma once\n")
+    output.write("#ifdef __cplusplus\n")
+    output.write("extern \"C\" {\n")
+    output.write("#endif\n\n")
+
+    output.write(f"extern unsigned char impeller_{args.symbol_name}_data[];\n")
+    output.write(f"extern unsigned long impeller_{args.symbol_name}_length;\n\n")
+
+    output.write("#ifdef __cplusplus\n")
+    output.write("}\n")
+    output.write("#endif\n")
+
+if __name__ == '__main__':
+  Main()

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,8 @@ impeller_component("playground") {`
`14`	`14`	`]`
`15`	`15`
`16`	`16`	`public_deps = [`
	`17`	`+ "../entity:entity_shaders",`
	`18`	`+ "../fixtures:shader_fixtures",`
`17`	`19`	`"../renderer",`
`18`	`20`	`"//flutter/testing",`
`19`	`21`	`"//third_party/glfw",`