Merge pull request intel#259 from wenjuan1-intel/shliclel180-testroot-topic

Enable llvm_test_suite_sycl ESIMD performance testing

Author: wenjuan1-intel

SYCL/ESIMD/esimd_test_utils.hpp

@@ -190,4 +190,13 @@ inline double report_time(const std::string &msg, event e0, event en) {
   return elapsed;
 }
 
+void display_timing_stats(double const kernelTime,
+                          unsigned int const uiNumberOfIterations,
+                          double const overallTime)
+{
+    std::cout << "Number of iterations: " << uiNumberOfIterations << "\n";
+    std::cout << "[KernelTime]:" << kernelTime << "\n";
+    std::cout << "[OverallTime][Primary]:" << overallTime << "\n";
+}
+
 } // namespace esimd_test

SYCL/ESIMD/histogram.cpp

@@ -132,11 +132,16 @@ int main(int argc, char *argv[]) {
                          image_channel_type::unsigned_int32,
                          range<2>{width / sizeof(uint4), height});
 
+  // Start Timer
+  esimd_test::Timer timer;
+  double start;
+
   // Launches the task on the GPU.
   double kernel_times = 0;
   unsigned num_iters = 10;
 
   try {
+    // num_iters + 1, iteration#0 is for warmup
     for (int iter = 0; iter <= num_iters; ++iter) {
       double etime = 0;
       for (int b = 0; b < NUM_BINS; b++)
@@ -215,6 +220,8 @@ int main(int argc, char *argv[]) {
       etime = esimd_test::report_time("kernel time", e, e);
       if (iter > 0)
         kernel_times += etime;
+      else
+        start = timer.Elapsed();
     }
     // SYCL will enqueue and run the kernel. Recall that the buffer's data is
     // given back to the host at the end of scope.
@@ -224,8 +231,12 @@ int main(int argc, char *argv[]) {
     return e.get_cl_code();
   }
 
-  float kernel_time = kernel_times / num_iters;
-  std::cerr << "GPU kernel time = " << kernel_time << " msec\n";
+  // End timer.
+  double end = timer.Elapsed();
+
+  esimd_test::display_timing_stats(kernel_times,num_iters,(end-start)*1000);
+  //float kernel_time = kernel_times / num_iters;
+  //std::cerr << "GPU kernel time = " << kernel_time << " msec\n";
 
   writeHist(bins);
   writeHist(cpuHistogram);

llvm_test_suite_sycl.pl

@@ -10,16 +10,15 @@
 
 # @test_to_run_list stores only the test(s) that will be run
 # For example, for "tc -t llvm_test_suite_sycl/aot_cpu,aot_gpu" it will store 2 tests - aot_cpu and aot_gpu
-my @test_to_run_list = ();
+my @test_to_run_list = get_tests_to_run();
 # @suite_test_list stores all the tests in the whole suite(without splitting) or sub-suite(with splitting)
 # For example, for "tc -t llvm_test_suite_sycl~4-1/aot_cpu,aot_gpu" it will store all the tests in sub-suite 4-1
-my @suite_test_list = ();
+my @suite_test_list = get_test_list($current_optset);
 my $short_test_name;
 my $test_info;
-my $config_folder = "";
+my $config_folder = 'config_sycl';
 my $subdir = "SYCL";
 my $insert_command = "";
-my $is_suite = 0;
 
 my $sycl_backend = "";
 my $device = "";
@@ -52,13 +51,27 @@ sub lscl {
     return $output;
 }
 
+sub is_zperf_run {
+    if ((defined $opt_perf && $opt_perf) ||
+        (defined $opt_perf_run && $opt_perf_run)) {
+        return 1;
+    }
+    return 0;
+}
+
+sub is_suite {
+    my @whole_suite_test = sort(@suite_test_list);
+    my @current_test_list = sort(@test_to_run_list);
+
+    return is_same(\@current_test_list, \@whole_suite_test);
+}
+
 sub init_test
 {
     my $suite_feature = $current_suite;
     $suite_feature =~ s/^llvm_test_suite_//;
     #Remove suffix of suite names if it has
     $suite_feature =~ s/~.*$//;
-    $config_folder = 'config_sycl';
     if ($suite_feature !~ /^sycl/)
     {
         $config_folder = $config_folder . '_' . $suite_feature;
@@ -93,7 +106,7 @@ sub init_test
     }
 
     #Remove untested source files from $subdir if it run with several subsuites
-    if ($is_suite) {
+    if (is_suite()) {
       my $info_dir = "$optset_work_dir/$config_folder";
       my @info_files = glob("$info_dir/*.info");
 
@@ -124,21 +137,69 @@ sub init_test
     return PASS;
 }
 
+sub extract_perf_results
+{
+    my $timer = Timer->new($current_test, $current_suite, $current_optset);
+    $timer->set("host", &alloy_utils::get_hostname());
+    my $output_file = join($slash, $optset_work_dir, "$current_test.output");
+    open(LOG, "+>", $output_file) or die "open $output_file fail";
+    print LOG $execution_output;
+    seek(LOG, 0, 0);
+    my $perf_matched = 0;
+    while (<LOG>) {
+        my $pattern = ".*OverallTime(.*):(\\d+.?\\d*[Ee]?[+-]?\\d+).*";
+        if ($_ =~ qr/$pattern/) {
+            my $primary = $1;
+            my $result = $2;
+            my $metric = "time";
+            my $better = "lt";
+            $timer->set($metric, $result);
+            $timer->set("BETTER_$metric", $better);
+            if ($primary =~ m/Primary/) {
+              $timer->set("primary_metric", $metric);
+              $perf_matched = 1;
+            }
+        }
+        $pattern = ".*KernelThroughput(.*):(\\d+.?\\d*[Ee]?[+-]?\\d+).*";
+        if ($_ =~ qr/$pattern/) {
+            my $primary = $1;
+            my $result = $2;
+            my $metric = "throughput";
+            my $better = "gt";
+            $timer->set($metric, $result);
+            $timer->set("BETTER_$metric", $better);
+            if ($primary =~ m/Primary/) {
+              $timer->set("primary_metric", $metric);
+              $perf_matched = 1;
+            }
+        }
+        $pattern = ".*KernelTime(.*):(\\d+.?\\d*[Ee]?[+-]?\\d+).*";
+        if ($_ =~ qr/$pattern/) {
+            my $primary = $1;
+            my $result = $2;
+            my $metric = "kerneltime";
+            my $better = "lt";
+            $timer->set($metric, $result);
+            $timer->set("BETTER_$metric", $better);
+            if ($primary =~ m/Primary/) {
+              $timer->set("primary_metric", $metric);
+              $perf_matched = 1;
+            }
+        }
+    }
+    close(LOG);
+    if (! $perf_matched) {
+        print "Warning: Primary metric is not specified!!!\n";
+    }
+}
+
 sub BuildTest
 {
     $build_dir = $cwd . "/build";
     safe_Mkdir($build_dir);
 
-    # CMPLRTST-14094: API get_test_list() requires optset name as an argument
-    @suite_test_list = get_test_list($current_optset);
-
-    @test_to_run_list = get_tests_to_run();
     if ($current_test eq $test_to_run_list[0])
     {
-        my @whole_suite_test = sort(@suite_test_list);
-        my @current_test_list = sort(@test_to_run_list);
-        $is_suite = is_same(\@current_test_list, \@whole_suite_test);
-
         init_test();
         chdir_log($build_dir);
 
@@ -168,6 +229,9 @@ sub BuildTest
 
 sub RunTest
 {
+    $build_dir = $cwd . "/build";
+    chdir_log($build_dir);
+
     $test_info = get_info();
     my ( $status, $output) = do_run($test_info);
     my $res = "";
@@ -180,6 +244,9 @@ sub RunTest
     } else {
         $res = generate_run_result($output);
     }
+    if ($res eq $PASS && is_zperf_run()) {
+        extract_perf_results();
+    }
     return $res;
 }
 
@@ -215,7 +282,7 @@ sub do_run
         $timeset = "--timeout 0";
       }
 
-      if ($is_suite) {
+      if (is_suite()) {
         set_tool_path();
         execute("$python $lit -a $matrix $jobset . $timeset > $run_all_lf 2>&1");
       } else {

llvm_test_suite_sycl.xml

@@ -3,7 +3,7 @@
   <description>Port of llvm_test_suite_sycl.
 Suite is autogenerated by suite_generator_sycl.pl that you can find in the root dir of suite
 Sources repo https://github.com/intel-innersource/applications.compilers.tests.llvm-project-llvm-test-suite
-Last Changed Revision: 4be68f9c03290a6b48ef040f402ea754ae4d2998    Tue Sep 21 16:16:04 2021 -0700
+Last Changed Revision: 83d34b70c59796d3d3504c6e1d56771115c89ae4    Tue Sep 21 16:16:04 2021 -0700
 </description>
   <files>
     <file path="cmake" />
@@ -17,6 +17,9 @@ Last Changed Revision: 4be68f9c03290a6b48ef040f402ea754ae4d2998    Tue Sep 21 16
     <file path="config_sycl" />
     <file path=".github/CODEOWNERS" />
   </files>
+  <rules>
+    <advancedRule perfSupport="accurate" />
+  </rules>
   <tests>
     <test configFile="config_sycl/TEMPLATE_llvm_test_suite_sycl.xml" splitGroup="basic" testName="Basic_opencl_interop_0" />
     <test configFile="config_sycl/TEMPLATE_llvm_test_suite_sycl.xml" splitGroup="aot" testName="aot_accelerator" />

llvm_test_suite_sycl_matrix.xml

@@ -3,7 +3,7 @@
   <description>Port of llvm_test_suite_sycl_matrix.
 Suite is autogenerated by suite_generator_sycl.pl that you can find in the root dir of suite
 Sources repo https://github.com/intel-innersource/applications.compilers.tests.llvm-project-llvm-test-suite
-Last Changed Revision: 4be68f9c03290a6b48ef040f402ea754ae4d2998    Tue Sep 21 16:16:04 2021 -0700
+Last Changed Revision: 83d34b70c59796d3d3504c6e1d56771115c89ae4    Tue Sep 21 16:16:04 2021 -0700
 </description>
   <files>
     <file path="cmake" />

llvm_test_suite_sycl_valgrind.xml

@@ -3,7 +3,7 @@
   <description>Port of llvm_test_suite_sycl.
 Suite is autogenerated by suite_generator_sycl.pl that you can find in the root dir of suite
 Sources repo https://github.com/intel-innersource/applications.compilers.tests.llvm-project-llvm-test-suite
-Last Changed Revision: 4be68f9c03290a6b48ef040f402ea754ae4d2998    Tue Sep 21 16:16:04 2021 -0700
+Last Changed Revision: 83d34b70c59796d3d3504c6e1d56771115c89ae4    Tue Sep 21 16:16:04 2021 -0700
 </description>
   <files>
     <file path="cmake" />
@@ -17,6 +17,9 @@ Last Changed Revision: 4be68f9c03290a6b48ef040f402ea754ae4d2998    Tue Sep 21 16
     <file path="config_sycl" />
     <file path=".github/CODEOWNERS" />
   </files>
+  <rules>
+    <advancedRule perfSupport="accurate" />
+  </rules>
   <tests>
     <test configFile="config_sycl/TEMPLATE_llvm_test_suite_sycl.xml" splitGroup="basic" testName="Basic_opencl_interop_0" />
     <test configFile="config_sycl/TEMPLATE_llvm_test_suite_sycl.xml" splitGroup="aot" testName="aot_accelerator" />