micro_benchmarks.py: Run the project from an isolated directory (#2330)

* micro_benchmarks.py: Run the project from an isolated directory

Problem:

When running `MicroBenchmarks.csproj` for wasm, the build does:

1. `dotnet restore /Users/radical/dev/performance/src/benchmarks/micro/MicroBenchmarks.csproj ...`
2. `dotnet build /Users/radical/dev/performance/src/benchmarks/micro/MicroBenchmarks.csproj ...`
   - which emits to `./artifacts/{obj, bin}/MicroBenchmarks/`
3. `dotnet run --project /Users/radical/dev/performance/src/benchmarks/micro/MicroBenchmarks.csproj`
   - and this runs from the artifacts dir

Then the running `MicroBenchmarks` tries to build the same project as a project reference from a generated wrapper project:

4. `// start dotnet restore`; `// start dotnet build ..`
   - And this emits warnings, and then errors:
```
warning MSB3026: Could not copy "/Users/radical/dev/performance/artifacts/obj/MicroBenchmarks/Release/net7.0/MicroBenchmarks.pdb" to "/Users/radical/dev/performance/artifacts/bin/MicroBenchmarks/Release/net7.0/MicroBenchmarks.pdb". Beginning retry 1 in 1000ms. The process cannot access the file '/Users/radical/dev/performance/artifacts/bin/MicroBenchmarks/Release/net7.0/MicroBenchmarks.pdb' because it is being used by another process.
```

- This is because the project is running from the same directory, as
  what the build in (4) tries to work with, which causes the build
  failure. If I use a different directory for (4), then there are no
  errors.

This fails the build. But BDN ignores that, and runs the build again
with `--no-dependencies`, which then skips the build of
`MicroBenchmarks.csproj`, and just builds the wrapper project, thus
succeeding.

Solution:

This commit prevents this by copying the build output to a separate
directory, and running it from there.

Fixes #2327

* Add a `--run-isolated` command line parameter

This will cause the build output to be in
`artifacts/bin/for-running/{project_name}`, instead of
`artifacts/bin/{project_name}`. And then instead of using `dotnet run
--project` to run, `dotnet exec {project_name.dll}` will be used.

* Fix windows build - don't assume .dll extension

* Use dotnet exec only when running isolated

* clean up the api a bit

* Bump bdn to 0.13.1.1740

* micro_benchmarks.py: add back '--' to the 'dotnet run' command line

Author: radical

scripts/benchmarks_ci.py

@@ -230,9 +230,10 @@ def __main(args: list) -> int:
     # dotnet --info
     dotnet.info(verbose=verbose)
 
+    bin_dir_to_use=micro_benchmarks.get_bin_dir_to_use(args.csprojfile, args.bin_directory, args.run_isolated)
     BENCHMARKS_CSPROJ = dotnet.CSharpProject(
         project=args.csprojfile,
-        bin_directory=args.bin_directory
+        bin_directory=bin_dir_to_use
     )
 
     if not args.run_only:
@@ -243,6 +244,7 @@ def __main(args: list) -> int:
             args.configuration,
             target_framework_monikers,
             args.incremental,
+            args.run_isolated,
             verbose
         )
 
@@ -255,6 +257,7 @@ def __main(args: list) -> int:
                     BENCHMARKS_CSPROJ,
                     args.configuration,
                     framework,
+                    args.run_isolated,
                     verbose,
                     args
                 )

scripts/dotnet.py

@@ -39,6 +39,18 @@ def info(verbose: bool) -> None:
     cmdline = ['dotnet', '--info']
     RunCommand(cmdline, verbose=verbose).run()
 
+def exec(asm_path: str, verbose: bool, *args) -> None:
+    """
+    Executes `dotnet exec` which can be used to execute assemblies
+    """
+    asm_path=path.abspath(asm_path)
+    working_dir=path.dirname(asm_path)
+    if not path.exists(asm_path):
+        raise ArgumentError('Cannot find assembly {} to exec'.format(asm_path))
+
+    cmdline = ['dotnet', 'exec', path.basename(asm_path)]
+    cmdline += list(args)
+    RunCommand(cmdline, verbose=verbose).run(working_dir)
 
 def __log_script_header(message: str):
     message_length = len(message)
@@ -620,6 +632,28 @@ def get_commit_date(
             'Could not get timestamp for commit %s' % commit_sha)
     return build_timestamp
 
+def get_project_name(csproj_file: str) -> str:
+    '''
+    Gets the project name from the csproj file path
+    '''
+    return path.splitext(path.basename(path.abspath(csproj_file)))[0]
+
+def get_main_assembly_path(
+        bin_directory: str,
+        project_name: str) -> str:
+    '''
+    Gets the main assembly path, as {project_name}.dll, or .exe
+    '''
+    exe=path.join(bin_directory, project_name + '.exe')
+    if path.exists(exe):
+        return exe
+
+    dll=path.join(bin_directory, project_name + '.dll')
+    if path.exists(dll):
+        return dll
+
+    raise ValueError(
+        'Unable to find main assembly - {} or {} in {}'.format(exe, dll, bin_directory))
 
 def get_build_directory(
         bin_directory: str,

scripts/micro_benchmarks.py

@@ -154,6 +154,15 @@ def __get_bdn_arguments(user_input: str) -> list:
              '''harness.''',
     )
 
+    parser.add_argument(
+        '--run-isolated',
+        dest='run_isolated',
+        required=False,
+        default=False,
+        action='store_true',
+        help='Move the binaries to a different directory for running',
+    )
+
     def __valid_dir_path(file_path: str) -> str:
         '''Verifies that specified file path exists.'''
         file_path = path.abspath(file_path)
@@ -222,7 +231,7 @@ def __process_arguments(args: list) -> Tuple[list, bool]:
 
 
 def __get_benchmarkdotnet_arguments(framework: str, args: tuple) -> list:
-    run_args = ['--']
+    run_args = []
     if args.corerun:
         run_args += ['--coreRun'] + args.corerun
     if args.cli:
@@ -260,12 +269,21 @@ def __get_benchmarkdotnet_arguments(framework: str, args: tuple) -> list:
 
     return run_args
 
+def get_bin_dir_to_use(csprojfile: dotnet.CSharpProjFile, bin_directory: str, run_isolated: bool) -> str:
+    '''
+    Gets the bin_directory, which might be different if run_isolate=True
+    '''
+    if run_isolated:
+        return path.join(bin_directory, 'for-running', dotnet.get_project_name(csprojfile.file_name))
+    else:
+        return bin_directory
 
 def build(
         BENCHMARKS_CSPROJ: dotnet.CSharpProject,
         configuration: str,
         target_framework_monikers: list,
         incremental: str,
+        run_isolated: bool,
         verbose: bool) -> None:
     '''Restores and builds the benchmarks'''
 
@@ -291,32 +309,49 @@ def build(
     BENCHMARKS_CSPROJ.build(
         configuration=configuration,
         target_framework_monikers=target_framework_monikers,
+        output_to_bindir=run_isolated,
         verbose=verbose,
         packages_path=packages)
 
+    # When running isolated, artifacts/obj/{project_name} will still be
+    # there, and would interfere with any subsequent builds. So, remove
+    # that
+    if run_isolated:
+        objDir = path.join(get_artifacts_directory(), 'obj', BENCHMARKS_CSPROJ.project_name)
+        remove_directory(objDir)
 
 def run(
         BENCHMARKS_CSPROJ: dotnet.CSharpProject,
         configuration: str,
         framework: str,
+        run_isolated: bool,
         verbose: bool,
         *args) -> None:
     '''Runs the benchmarks'''
     __log_script_header("Running .NET micro benchmarks for '{}'".format(
         framework
     ))
-    # dotnet run
+
+    # dotnet exec
     run_args = __get_benchmarkdotnet_arguments(framework, *args)
     target_framework_moniker = dotnet.FrameworkAction.get_target_framework_moniker(
         framework
     )
-    BENCHMARKS_CSPROJ.run(
-        configuration,
-        target_framework_moniker,
-        verbose,
-        *run_args
-    )
-            
+
+    if run_isolated:
+        runDir = BENCHMARKS_CSPROJ.bin_path
+        asm_path=dotnet.get_main_assembly_path(runDir, BENCHMARKS_CSPROJ.project_name)
+        dotnet.exec(asm_path, verbose, *run_args)
+    else:
+        # This is needed for `dotnet run`, but not for `dotnet exec`
+        run_args = ['--'] + run_args
+        BENCHMARKS_CSPROJ.run(
+            configuration,
+            target_framework_moniker,
+            verbose,
+            *run_args
+        )
+
 def __log_script_header(message: str):
     getLogger().info('-' * len(message))
     getLogger().info(message)
@@ -340,9 +375,10 @@ def __main(args: list) -> int:
         # dotnet --info
         dotnet.info(verbose)
 
+        bin_dir_to_use=micro_benchmarks.get_bin_dir_to_use(args.csprojfile, args.bin_directory, args.run_isolated)
         BENCHMARKS_CSPROJ = dotnet.CSharpProject(
             project=args.csprojfile,
-            bin_directory=args.bin_directory
+            bin_directory=bin_dir_to_use
         )
 
         # dotnet build
@@ -351,6 +387,7 @@ def __main(args: list) -> int:
             configuration,
             target_framework_monikers,
             incremental,
+            args.run_isolated,
             verbose
         )
 
@@ -360,6 +397,7 @@ def __main(args: list) -> int:
                 BENCHMARKS_CSPROJ,
                 configuration,
                 framework,
+                args.run_isolated,
                 verbose,
                 args
             )