Merge pull request swiftlang#96 from graydon/misc-run_cperf-fixes

Misc run cperf fixes

Author: graydon

run_cperf

@@ -29,8 +29,13 @@ NEW_INSTANCE = 'new'
 
 
 def get_workspace_for_instance(instance, args):
-    return common.private_workspace(
-        os.path.join(args.swift_branch, instance))
+    if instance == OLD_INSTANCE and args.old_workspace is not None:
+        return args.old_workspace
+    elif instance == NEW_INSTANCE and args.new_workspace is not None:
+        return args.new_workspace
+    else:
+        return common.private_workspace(
+            os.path.join(args.swift_branch, instance))
 
 
 def setup_workspace(instance, workspace, args):
@@ -84,6 +89,7 @@ def main():
         with open(ws_comment, 'w') as f:
             f.write("Compilation-performance test failed")
 
+    failures = []
     for instance in instances:
         workspace = get_workspace_for_instance(instance, args)
 
@@ -95,9 +101,9 @@ def main():
             build_swift_toolchain(workspace, args)
 
         if not args.skip_runner:
-            execute_runner(instance, workspace, configs, args)
+            failures += execute_runner(instance, workspace, configs, args)
 
-    regressions = analyze_results(configs, args)
+    regressions = analyze_results(configs, failures, args)
 
     # Crude hack to write output to workspace when in CI,
     # regardless of --output passed.
@@ -143,8 +149,12 @@ def get_sandbox_profile_flags():
     return sandbox_flags
 
 
-def get_swiftc_path(workspace):
-    if platform.system() == 'Darwin':
+def get_swiftc_path(instance, workspace, args):
+    if instance == OLD_INSTANCE and args.old_swiftc is not None:
+        return args.old_swiftc
+    elif instance == NEW_INSTANCE and args.new_swiftc is not None:
+        return args.new_swiftc
+    elif platform.system() == 'Darwin':
         swiftc_path = os.path.join(workspace, 'build/compat_macos/install/toolchain/usr/bin/swiftc')
     elif platform.system() == 'Linux':
         swiftc_path = os.path.join(workspace, 'build/compat_linux/install/usr/bin/swiftc')
@@ -245,7 +255,7 @@ def get_stats_dir(instance, variant):
 
 
 def get_actual_config_and_flags(config, stats):
-    flags = ("-stats-output-dir '%s'" % stats)
+    flags = ("-stats-output-dir  " + stats)
     # Handle pseudo-configs
     if config == 'wmo-onone':
         flags += ' -wmo -Onone '
@@ -262,29 +272,41 @@ def get_variant(config, args):
 
 def execute_runner(instance, workspace, configs, args):
     projects = get_projects(args.suite)
-    swiftc_path = get_swiftc_path(workspace)
+    swiftc_path = get_swiftc_path(instance, workspace, args)
+    failures = []
     for config in configs:
         variant = get_variant(config, args)
         stats = get_stats_dir(instance, variant)
         if os.path.exists(stats):
             shutil.rmtree(stats)
         os.makedirs(stats)
         (config, flags) = get_actual_config_and_flags(config, stats)
+        actions = 'action.startswith("BuildSwiftPackage")'
+        if platform.system() == 'Darwin':
+            # Darwin can handle xcodebuild as well
+            actions = 'action.startswith("Build")'
         runner_command = [
             './runner.py',
             '--swiftc', swiftc_path,
             '--projects', projects,
             '--build-config', config,
             '--swift-version', '3',
-            '--include-actions', 'action.startswith("Build")',
+            '--include-actions', actions,
             '--swift-branch', args.swift_branch,
             '--add-swift-flags', flags,
         ]
         if args.sandbox:
             runner_command += get_sandbox_profile_flags()
         if args.verbose:
             runner_command += ["--verbose"]
-        common.check_execute(runner_command, timeout=9999999)
+        try:
+            common.check_execute(runner_command, timeout=9999999)
+        except common.ExecuteCommandFailure:
+            fail_logs = [ re.sub('FAIL_.*_(\\w+).log', '\\1', n)
+                          for n in os.listdir(".")
+                          if re.match('FAIL.*.log', n)]
+            failures += (fail_logs if len(fail_logs) != 0 else ["unknown"])
+    return failures
 
 
 def get_table_name(reference, subset, variant):
@@ -328,7 +350,7 @@ def get_baseline_name(variant):
                         'cperf-baselines', variant + '.csv')
 
 
-def analyze_results(configs, args):
+def analyze_results(configs, failures, args):
     # Make a separate random run_id for each comment, to uniquify links.
     run_id = hex(random.randint(1, 2**63))[2:]
     old_ws = get_workspace_for_instance(OLD_INSTANCE, args)
@@ -383,6 +405,9 @@ def analyze_results(configs, args):
     out = args.output
     regressions = any(x != 0 for x in returncodes)
     out.write("# Summary for %s %s\n\n" % (args.swift_branch, args.suite))
+    if len(failures) != 0:
+        out.write("**Unexpected test results, stats may be off for %s**\n\n" %
+                  ", ".join(set(failures)))
     if regressions:
         out.write("**Regressions found (see below)**\n\n")
     else:
@@ -477,6 +502,14 @@ def parse_args():
                         action='store_true')
     parser.add_argument('--setup-workspaces-for-pr',
                         type=int, default=None)
+    parser.add_argument('--old-workspace',
+                        type=str, default=None)
+    parser.add_argument('--new-workspace',
+                        type=str, default=None)
+    parser.add_argument('--old-swiftc',
+                        type=str, default=None)
+    parser.add_argument('--new-swiftc',
+                        type=str, default=None)
     return parser.parse_args()