update one product build package logic (#423)

Author: cynthiajoan

.github/workflows/package.yml

@@ -41,6 +41,7 @@ on:
 env:
   # Use SHA256 for hashing files.
   hashCommand: "sha256sum"
+  default_apis: 'analytics,auth,crashlytics,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage'
 
 jobs:
   package_sdks:
@@ -49,6 +50,23 @@ jobs:
     strategy:
       fail-fast: false
     steps:
+      - name: Check input
+        # This check the apis input. It detect whether the package is only asking for subset of
+        # of the products, like auth,firestore only etc. If the input apis are less than the
+        # default_apis env attribute, we package only the provided apis, for test purposes only.
+        id: check-input
+        shell: bash
+        run: |
+          IFS=',' read -r -a input_apis <<< "${{ inputs.apis }}"
+          IFS=',' read -r -a default_apis <<< "${{ env.default_apis }}"
+          if [[ ${#input_apis[@]} != ${#default_apis[@]} ]]; then
+            echo "::set-output name=package_for_checks::1"
+            echo "::set-output name=package_apis::'--apis=${{ inputs.apis }}'"
+          else
+            echo "::set-output name=package_for_checks::0"
+            echo "::set-output name=package_apis::"
+          fi
+
       - name: Print inputs
         shell: bash
         run: |
@@ -63,6 +81,8 @@ jobs:
           echo download_windows_run: ${{ inputs.download_windows_run }}
           echo platforms: ${{ inputs.platforms }}"
           echo apis: ${{ inputs.apis }}"
+          echo is the package for checks: ${{ steps.check-input.outputs.package_for_checks }}
+          echo package_apis: ${{ steps.check-input.outputs.package_apis }}
 
       - name: Check out base branch
         uses: actions/[email protected]
@@ -135,7 +155,7 @@ jobs:
 
       - name: Package unitypackage
         run: |
-          python scripts/build_scripts/build_package.py --zip_dir=built_artifact
+          python scripts/build_scripts/build_package.py --zip_dir=built_artifact ${{ steps.check-input.outputs.package_apis }}
 
       - name: Commit Changes if there is any
         if: inputs.working_branch != '' || inputs.create_new_branch == '1'
@@ -210,15 +230,18 @@ jobs:
           path: firebase_unity_sdk.zip
 
       - name: Package tgz
+        if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
         run: |
           python scripts/build_scripts/build_package.py --zip_dir=built_artifact --output_upm=True --output=output_tgz
 
       - name: Listing output tgz
+        if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
         run: |
           ls -Rl
         working-directory: output_tgz
 
       - name: compute SDK hash for tgz files
+        if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
         shell: bash
         run: |
           tgz_files_list=$(find -type f -name '*.tgz')
@@ -229,6 +252,7 @@ jobs:
         working-directory: output_tgz
 
       - name: Upload Build tgz
+        if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
         uses: actions/upload-artifact@v2
         with:
           name: firebase_unity_sdk_tgz

scripts/build_scripts/build_package.py

@@ -39,12 +39,18 @@
 from absl import flags
 from absl import logging
 
+
+SUPPORT_TARGETS = [
+    "analytics", "auth", "crashlytics", "database", "dynamic_links",
+    "firestore", "functions", "installations", "messaging", "remote_config",
+    "storage"
+]
+
 FLAGS = flags.FLAGS
 flags.DEFINE_string('zip_dir', None,
                     'Directory of zip files from build output to package')
 flags.DEFINE_string("config_file", "exports.json",
-                    ("Config file that describes how to "
-                     "pack the unity assets."))
+                    ("Config file that describes how to pack the unity assets."))
 flags.DEFINE_string("guids_file", "guids.json",
                     "Json file with stable guids cache.")
 
@@ -55,7 +61,13 @@
                     "Output folder for unitypackage.")
 
 flags.DEFINE_boolean("output_upm", False, "Whether output packages as tgz for"
-    "Unity Package Manager.")
+                     "Unity Package Manager.")
+
+flags.DEFINE_string("apis", None, "which firebase products to pack, if not"
+                    "set means package all. Value should be items in [{}],"
+                    "connected with ',', eg 'auth,firestore'".format(
+                        ",".join(SUPPORT_TARGETS)))
+
 
 def get_zip_files():
   """Get all zip files from FLAGS.zip_dir.
@@ -75,12 +87,14 @@ def get_zip_files():
 
   return zip_file_paths
 
+
 def get_last_version():
   """Get the last version number in guids file if exists.
     Returns:
       version number.
   """
-  version_cmake_path = os.path.join(os.getcwd(), "cmake", "firebase_unity_version.cmake")
+  version_cmake_path = os.path.join(
+      os.getcwd(), "cmake", "firebase_unity_version.cmake")
   with open(version_cmake_path, "r") as f:
     datafile = f.readlines()
     for line in datafile:
@@ -89,45 +103,68 @@ def get_last_version():
         return result[-1].strip("\"")
   return None
 
+
 def find_pack_script():
   """Get the pack script either from intermediate build folder or download from unity-jar-resolver.
 
     Returns:
       path of the pack script. None if not found.
   """
   built_folder_ext = "_unity"
-  built_folder_postion = os.path.join("external", "src", "google_unity_jar_resolver")
+  built_folder_postion = os.path.join(
+      "external", "src", "google_unity_jar_resolver")
   built_folder = None
   resolver_root_folder = "unity-jar-resolver"
   for folder in os.listdir("."):
     if folder.endswith(built_folder_ext):
       built_folder = folder
       break
-  
+
   if built_folder != None:
     resolver_root_folder = os.path.join(built_folder, built_folder_postion)
   elif not os.path.exists(resolver_root_folder):
     git_clone_script = ["git", "clone",
-      "--depth", "1",
-      "https://github.com/googlesamples/unity-jar-resolver.git"]
+                        "--depth", "1",
+                        "https://github.com/googlesamples/unity-jar-resolver.git"]
     subprocess.call(git_clone_script)
 
   if resolver_root_folder != None:
-    script_path = os.path.join(resolver_root_folder, "source", "ExportUnityPackage", "export_unity_package.py")
+    script_path = os.path.join(
+        resolver_root_folder, "source", "ExportUnityPackage", "export_unity_package.py")
     return script_path
   return None
 
+
+def _debug_create_target_package(target, packer_script_path, guids_file_path, output_folder, zip_file_list, last_version):
+  debug_config_path = os.path.join(os.getcwd(), FLAGS.script_folder, "debug_single_export_json",
+                                   target+".json")
+  debug_cmd_args = [
+      sys.executable,
+      packer_script_path,
+      "--assets_dir=" + FLAGS.zip_dir,
+      "--config_file=" + debug_config_path,
+      "--guids_file=" + guids_file_path,
+      "--output_dir=" + output_folder,
+      "--output_upm=False",
+  ]
+  debug_cmd_args.extend(
+      ["--assets_zip=" + zip_file for zip_file in zip_file_list])
+  debug_cmd_args.append("--enabled_sections=build_dotnet4 asset_package_only")
+  debug_cmd_args.append("--plugins_version=" + last_version)
+  subprocess.call(debug_cmd_args)
+  logging.info("Debug Packaging done for target %s", target)
+
+
 def main(argv):
   if len(argv) > 1:
     raise app.UsageError('Too many command-line arguments.')
 
   packer_script_path = find_pack_script()
   if packer_script_path == None:
-    raise app.UsageError('Cannot find pack script. Please build the project first.')
-   
+    raise app.UsageError(
+        'Cannot find pack script. Please build the project first.')
+
   packer_script_path = os.path.join(os.getcwd(), packer_script_path)
-  config_file_path = os.path.join(os.getcwd(), FLAGS.script_folder,
-                                  FLAGS.config_file)
   guids_file_path = os.path.join(os.getcwd(), FLAGS.script_folder,
                                  FLAGS.guids_file)
 
@@ -141,6 +178,21 @@ def main(argv):
   if os.path.exists(output_folder):
     shutil.rmtree(output_folder)
 
+  if FLAGS.apis:
+    # If told to only build a subset, package just those products and exit early.
+    api_list = FLAGS.apis.split(",")
+    if not set(api_list).issubset(set(SUPPORT_TARGETS)):
+      raise app.UsageError("apis parameter error, Value should be items in [{}],"
+                           "connected with ',', eg 'auth,firestore'".format(
+                               ",".join(SUPPORT_TARGETS)))
+
+    for target in api_list:
+      _debug_create_target_package(
+          target, packer_script_path, guids_file_path, output_folder, zip_file_list, last_version)
+    return
+
+  config_file_path = os.path.join(os.getcwd(), FLAGS.script_folder,
+                                  FLAGS.config_file)
   cmd_args = [
       sys.executable,
       packer_script_path,
@@ -151,16 +203,17 @@ def main(argv):
       "--output_upm=" + str(FLAGS.output_upm),
   ]
   cmd_args.extend(["--assets_zip=" + zip_file for zip_file in zip_file_list])
-  
+
   if last_version:
     cmd_args.append("--plugins_version=" + last_version)
 
   if FLAGS.output_upm:
     cmd_args.append("--enabled_sections=build_dotnet4")
     cmd_args.append("--output_unitypackage=False")
   else:
-    cmd_args.append("--enabled_sections=build_dotnet3 build_dotnet4 asset_package_only")
-  
+    cmd_args.append(
+        "--enabled_sections=build_dotnet3 build_dotnet4 asset_package_only")
+
   # Check if need to gen new guids
   p = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   output, error = p.communicate()
@@ -170,28 +223,30 @@ def main(argv):
     error_str = error_str.split("assets:")[-1]
     error_str = error_str.rstrip("\\n\'")
     split_string = error_str.split(" ")
-    split_string = split_string[3:] # exclude first 3 lines
-    gen_guids_script_path = os.path.join(os.getcwd(), "scripts", "build_scripts", "gen_guids.py")
+    split_string = split_string[3:]  # exclude first 3 lines
+    gen_guids_script_path = os.path.join(
+        os.getcwd(), "scripts", "build_scripts", "gen_guids.py")
     gen_cmd_args = [
-      sys.executable,
-      gen_guids_script_path,
-      "--guids_file=" + guids_file_path,
-      "--version=" + last_version,
-      "--generate_new_guids=True",
+        sys.executable,
+        gen_guids_script_path,
+        "--guids_file=" + guids_file_path,
+        "--version=" + last_version,
+        "--generate_new_guids=True",
     ]
     for file in split_string:
-      file=file.strip("\"")
+      file = file.strip("\"")
       print(file)
       gen_cmd_args.append(file)
     subprocess.call(gen_cmd_args)
 
     # Need to package again if has that error
-    subprocess.call(cmd_args)   
+    subprocess.call(cmd_args)
   else:
     logging.info("No new guid generated.")
     error_list = str(error).split("\\n")
     logging.info("\n".join(error_list))
   logging.info("Packaging done for version %s", last_version)
 
+
 if __name__ == '__main__':
   app.run(main)

scripts/create_debug_export.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Based on the unity_packer/exports.json file, generate separate debug
+export config for each product.
+
+Example usage:
+  python scripts/build_scripts/create_debug_export.py
+"""
+
+import os
+import json
+
+from absl import app
+from absl import flags
+from absl import logging
+
+API_PACKAGE_MAP = {
+    "analytics": "FirebaseAnalytics.unitypackage",
+    "auth": "FirebaseAuth.unitypackage",
+    "crashlytics": "FirebaseCrashlytics.unitypackage",
+    "database": "FirebaseDatabase.unitypackage",
+    "dynamic_links": "FirebaseDynamicLinks.unitypackage",
+    "firestore": "FirebaseFirestore.unitypackage",
+    "functions": "FirebaseFunctions.unitypackage",
+    "installations": "FirebaseInstallations.unitypackage",
+    "messaging": "FirebaseMessaging.unitypackage",
+    "remote_config": "FirebaseRemoteConfig.unitypackage",
+    "storage": "FirebaseStorage.unitypackage",
+}
+
+default_package_names = [
+    "FirebaseApp.unitypackage", "SampleCommon.unitypackage"]
+
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('prod_export', "exports.json",
+                    'Production export json template to copy from')
+flags.DEFINE_string("json_folder", "unity_packer",
+                    ("The root folder for json configs."))
+flags.DEFINE_string("output_folder", "debug_single_export_json",
+                    "Json file with stable guids cache.")
+
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError('Too many command-line arguments.')
+  prod_export_json_path = os.path.join(
+      os.getcwd(), FLAGS.json_folder, FLAGS.prod_export)
+
+  with open(prod_export_json_path, "r") as fin:
+    export_json = json.load(fin)
+
+    # making sure only dotnet4 exists
+    builds = export_json["builds"]
+    for idx, dict in enumerate(builds):
+      if dict["name"] == "dotnet3":
+        builds.pop(idx)
+
+    packages = export_json["packages"]
+
+    for api_name, package_name in API_PACKAGE_MAP.items():
+      output_path = os.path.join(
+          os.getcwd(), FLAGS.json_folder, FLAGS.output_folder, api_name + ".json")
+      output_dict = {}
+      output_package_list = []
+      for idx, package_dict in enumerate(packages):
+        if package_dict["name"] in default_package_names:
+          output_package_list.append(packages[idx])
+        elif package_dict["name"] == package_name:
+          output_package_list.append(packages[idx])
+      output_dict["packages"] = output_package_list
+      output_dict["builds"] = builds
+
+      with open(output_path, 'w', encoding='utf-8') as fout:
+        fout.write(json.dumps(output_dict, indent=2))
+      logging.info("Write %s", output_path)
+
+
+if __name__ == '__main__':
+  app.run(main)