Shader analysis with malioc (#39005)

Author: zanderso

ci/licenses_golden/excluded_files

@@ -149,6 +149,7 @@
 ../../../flutter/impeller/tessellator/tessellator_unittests.cc
 ../../../flutter/impeller/tools/build_metal_library.py
 ../../../flutter/impeller/tools/check_licenses.py
+../../../flutter/impeller/tools/malioc_diff.py
 ../../../flutter/impeller/tools/xxd.py
 ../../../flutter/impeller/typographer/typographer_unittests.cc
 ../../../flutter/lib/snapshot/libraries.json

impeller/fixtures/BUILD.gn

@@ -7,6 +7,10 @@ import("//flutter/testing/testing.gni")
 
 impeller_shaders("shader_fixtures") {
   name = "fixtures"
+
+  # Not analyzing because they are not performance critical, and mipmap uses
+  # textureLod, which uses an extension that malioc does not support.
+  analyze = false
   shaders = [
     "array.frag",
     "array.vert",

impeller/playground/imgui/BUILD.gn

@@ -6,6 +6,10 @@ import("//flutter/impeller/tools/impeller.gni")
 
 impeller_shaders("imgui_shaders") {
   name = "imgui"
+
+  # Not analyzing because they are not performance critical, and mipmap uses
+  # textureLod, which uses an extension that malioc does not support.
+  analyze = false
   shaders = [
     "imgui_raster.vert",
     "imgui_raster.frag",

impeller/tools/BUILD.gn

@@ -0,0 +1,31 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/compiled_action.gni")
+import("//flutter/common/config.gni")
+import("//flutter/impeller/tools/malioc.gni")
+import("//flutter/testing/testing.gni")
+
+declare_args() {
+  # Maximum number of malioc processes to run in parallel.
+  #
+  # To avoid out-of-memory errors we explicitly reduce the number of jobs.
+  impeller_concurrent_malioc_jobs = -1
+}
+
+if (impeller_concurrent_malioc_jobs == -1) {
+  _script = "//flutter/build/get_concurrent_jobs.py"
+  _args = [
+    "--reserve-memory=1GB",
+    "--memory-per-job",
+    "malioc=100MB",
+  ]
+  _concurrent_jobs = exec_script(_script, _args, "json", [ _script ])
+  impeller_concurrent_malioc_jobs = _concurrent_jobs.malioc
+  assert(impeller_concurrent_malioc_jobs > 0)
+}
+
+pool("malioc_pool") {
+  depth = impeller_concurrent_malioc_jobs
+}

impeller/tools/impeller.gni

@@ -4,6 +4,7 @@
 
 import("//build/compiled_action.gni")
 import("//flutter/common/config.gni")
+import("//flutter/impeller/tools/malioc.gni")
 import("//flutter/testing/testing.gni")
 
 declare_args() {
@@ -467,6 +468,7 @@ template("blobcat_library") {
 template("impeller_shaders_gles") {
   assert(defined(invoker.shaders), "Impeller shaders must be specified.")
   assert(defined(invoker.name), "Name of the shader library must be specified.")
+  assert(defined(invoker.analyze), "Whether to analyze must be specified.")
 
   shaders_base_name = string_join("",
                                   [
@@ -494,10 +496,23 @@ template("impeller_shaders_gles") {
     defines = [ "IMPELLER_TARGET_OPENGLES" ]
   }
 
+  gles_shaders =
+      filter_include(get_target_outputs(":$impellerc_gles"), [ "*.gles" ])
+
+  if (invoker.analyze) {
+    analyze_lib = "analyze_$target_name"
+    malioc_analyze_shaders(analyze_lib) {
+      shaders = gles_shaders
+      if (defined(invoker.gles_language_version)) {
+        gles_language_version = invoker.gles_language_version
+      }
+      deps = [ ":$impellerc_gles" ]
+    }
+  }
+
   gles_lib = "genlib_$target_name"
   blobcat_library(gles_lib) {
-    shaders =
-        filter_include(get_target_outputs(":$impellerc_gles"), [ "*.gles" ])
+    shaders = gles_shaders
     deps = [ ":$impellerc_gles" ]
   }
 
@@ -519,6 +534,10 @@ template("impeller_shaders_gles") {
   group(target_name) {
     public_deps = [ ":$embed_gles_lib" ]
 
+    if (invoker.analyze) {
+      public_deps += [ ":$analyze_lib" ]
+    }
+
     if (!impeller_enable_metal && !impeller_enable_vulkan) {
       public_deps += [ ":$reflect_gles" ]
     }
@@ -589,6 +608,10 @@ template("impeller_shaders") {
   }
 
   if (impeller_enable_opengles) {
+    analyze = true
+    if (defined(invoker.analyze) && !invoker.analyze) {
+      analyze = false
+    }
     gles_shaders = "gles_$target_name"
     impeller_shaders_gles(gles_shaders) {
       name = invoker.name
@@ -600,6 +623,7 @@ template("impeller_shaders") {
       } else {
         shaders = invoker.shaders
       }
+      analyze = analyze
     }
   }
 

impeller/tools/malioc.gni

@@ -0,0 +1,124 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/compiled_action.gni")
+import("//flutter/common/config.gni")
+import("//flutter/testing/testing.gni")
+
+declare_args() {
+  # Path to the Mali offline compiler tool 'malioc'.
+  impeller_malioc_path = ""
+
+  impeller_malioc_cores = []
+}
+
+if (impeller_malioc_path != "" && impeller_malioc_cores == []) {
+  core_list_file = "$root_gen_dir/mali_core_list.json"
+  exec_script("//build/gn_run_binary.py",
+              [
+                rebase_path(impeller_malioc_path, root_build_dir),
+                "--list",
+                "--format",
+                "json",
+                "--output",
+                rebase_path(core_list_file),
+              ])
+  _mali_cores = read_file(core_list_file, "json")
+  foreach(mali_core, _mali_cores.cores) {
+    impeller_malioc_cores += [ mali_core.core ]
+  }
+}
+
+template("malioc_analyze_shaders") {
+  # TODO(zra): Check that gles_language_version is in the supported set. For now
+  # assume that if it is set, it is being set to 460, which malioc does not
+  # support.
+  if (impeller_malioc_path == "" || defined(invoker.gles_language_version)) {
+    if (defined(invoker.gles_language_version) &&
+        invoker.gles_language_version != "460") {
+      print("Disabling analysis for shaders in $target_name due to gles",
+            "version explicitly set to ${invoker.gles_language_version}.")
+    }
+    group(target_name) {
+      not_needed(invoker, "*")
+    }
+  } else {
+    target_deps = []
+    foreach(core, impeller_malioc_cores) {
+      foreach(source, invoker.shaders) {
+        shader_file_name = get_path_info(source, "name")
+        analysis_target = "${target_name}_${shader_file_name}_${core}_malioc"
+        target_deps += [ ":$analysis_target" ]
+        action(analysis_target) {
+          forward_variables_from(invoker,
+                                 "*",
+                                 [
+                                   "args",
+                                   "depfile",
+                                   "inputs",
+                                   "outputs",
+                                   "pool",
+                                   "script",
+                                 ])
+
+          script = "//build/gn_run_binary.py"
+          pool = "//flutter/impeller/tools:malioc_pool"
+
+          # Should be "gles" or "vkspv"
+          backend_ext = get_path_info(source, "extension")
+          assert(backend_ext == "gles",
+                 "Shader for unsupported backend passed to malioc: {{source}}")
+
+          # Nest all malioc output under its own subdirectory of root_gen_dir
+          # so that it's easier to diff it against the state before any changes.
+          subdir = rebase_path(target_gen_dir, root_gen_dir)
+          output_file =
+              "$root_gen_dir/malioc/$subdir/${shader_file_name}.$core.json"
+          outputs = [ output_file ]
+
+          # Determine the kind of the shader from the file name
+          name = get_path_info(source, "name")
+          shader_kind_ext = get_path_info(name, "extension")
+
+          if (shader_kind_ext == "comp") {
+            shader_kind_flag = "--compute"
+          } else if (shader_kind_ext == "frag") {
+            shader_kind_flag = "--fragment"
+          } else if (shader_kind_ext == "geom") {
+            shader_kind_flag = "--geometry"
+          } else if (shader_kind_ext == "tesc") {
+            shader_kind_flag = "--tessellation_control"
+          } else if (shader_kind_ext == "tese") {
+            shader_kind_flag = "--tessellation_evaluation"
+          } else if (shader_kind_ext == "vert") {
+            shader_kind_flag = "--vertex"
+          } else {
+            assert(false, "Unknown shader kind: {{source}}")
+          }
+
+          args = [
+            rebase_path(impeller_malioc_path, root_build_dir),
+            "--format",
+            "json",
+            shader_kind_flag,
+            "--core",
+            core,
+            "--output",
+            rebase_path(output_file),
+          ]
+
+          if (backend_ext == "vkspv") {
+            args += [ "--vulkan" ]
+          }
+
+          args += [ rebase_path(source) ]
+        }
+      }
+    }
+
+    group(target_name) {
+      deps = target_deps
+    }
+  }
+}